cobweb-launcher 1.1.12__tar.gz → 1.1.14__tar.gz

{cobweb-launcher-1.1.12/cobweb_launcher.egg-info → cobweb-launcher-1.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.12
+Version: 1.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-1.1.14/cobweb/crawlers/__init__.py

@@ -0,0 +1,2 @@
+from .base_crawler import Crawler
+from .file_crawler import FileCrawlerAir

cobweb-launcher-1.1.14/cobweb/crawlers/base_crawler.py

@@ -0,0 +1,128 @@
+import threading
+
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.utils import download_log_info
+from cobweb import setting
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            upload_queue: Queue,
+            custom_func: Union[Mapping[str, Callable]],
+            launcher_queue: Union[Mapping[str, Queue]],
+    ):
+        super().__init__()
+
+        self.upload_queue = upload_queue
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.launcher_queue = launcher_queue
+
+        self.spider_thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        stream = True if setting.DOWNLOAD_MODEL else False
+        yield Request(seed.url, seed, stream=stream, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        pass
+
+    def get_seed(self) -> Seed:
+        return self.launcher_queue['todo'].pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self.upload_queue.push(item)
+        elif isinstance(item, Seed):
+            self.launcher_queue['new'].push(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self.launcher_queue['todo'].push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self.launcher_queue['done'].push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.identifier = DealModel.fail
+            self.launcher_queue['done'].push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while True:
+            seed = self.get_seed()
+
+            if not seed:
+                continue
+
+            elif seed.params.retry >= self.max_retries:
+                seed.params.identifier = DealModel.fail
+                self.launcher_queue['done'].push(seed)
+                continue
+
+            seed_detail_log_info = download_log_info(seed.to_dict)
+
+            request_iterators = self.request(seed)
+
+            if not isgenerator(request_iterators):
+                raise TypeError("request function isn't a generator!")
+
+            for request_item in request_iterators:
+
+                if isinstance(request_item, BaseItem):
+                    self.upload_queue.push(request_item)
+
+                elif isinstance(request_item, Request):
+                    try:
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+                        for download_item in download_iterators:
+                            if isinstance(download_item, Response):
+                                response_detail_log_info = download_log_info(download_item.to_dict)
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=response_detail_log_info
+                                ))
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    except Exception as e:
+                        logger.info(LogTemplate.download_exception.format(
+                            detail=seed_detail_log_info,
+                            retry=seed.params.retry,
+                            priority=seed.params.priority,
+                            seed_version=seed.params.seed_version,
+                            identifier=seed.identifier or "", exception=e
+                        ))
+                        seed.params.retry += 1
+                        self.launcher_queue['todo'].push(seed)
+
+    def run(self):
+        for index in range(self.spider_thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

cobweb-launcher-1.1.14/cobweb/crawlers/file_crawler.py

@@ -0,0 +1,98 @@
+import os
+from typing import Union
+from cobweb import setting
+from cobweb.utils import OssUtil
+from cobweb.crawlers import Crawler
+from cobweb.base import Seed, BaseItem, Request, Response
+from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
+
+
+oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
+
+
+class FileCrawlerAir(Crawler):
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        seed_dict = item.seed.to_dict
+        seed_dict["bucket_name"] = oss_util.bucket
+        try:
+            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
+
+            if oss_util.exists(key):
+                seed_dict["data_size"] = oss_util.head(key).content_length
+                yield Response(item.seed, "exists", **seed_dict)
+
+            else:
+                seed_dict.setdefault("end", "")
+                seed_dict.setdefault("start", 0)
+
+                if seed_dict["end"] or seed_dict["start"]:
+                    start, end = seed_dict["start"], seed_dict["end"]
+                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+                if not item.seed.identifier:
+                    content = b""
+                    chunk_size = oss_util.chunk_size
+                    min_upload_size = oss_util.min_upload_size
+                    seed_dict.setdefault("position", 1)
+
+                    response = item.download()
+
+                    content_type = response.headers.get("content-type", "").split(";")[0]
+                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
+
+                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                        """过滤响应文件类型"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = f"response content type is {content_type}"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
+                        """过小文件标识返回"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = "file size is too small"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
+                        """小文件直接下载"""
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                        response.close()
+                        oss_util.put(key, content)
+                        yield Response(item.seed, response, **seed_dict)
+
+                    else:
+                        """中大文件同步分片下载"""
+                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
+
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                            if len(content) >= chunk_size:
+                                upload_data = content[:chunk_size]
+                                content = content[chunk_size:]
+                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                                seed_dict['start'] += len(upload_data)
+                                seed_dict['position'] += 1
+
+                        response.close()
+
+                        if content:
+                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                        oss_util.merge(key, seed_dict["upload_id"])
+                        seed_dict["data_size"] = oss_util.head(key).content_length
+                        yield Response(item.seed, response, **seed_dict)
+
+                elif item.seed.identifier == "merge":
+                    oss_util.merge(key, seed_dict["upload_id"])
+                    seed_dict["data_size"] = oss_util.head(key).content_length
+                    yield Response(item.seed, "merge", **seed_dict)
+
+        except OssDBPutPartError:
+            yield Seed(seed_dict)
+        except OssDBMergeError:
+            yield Seed(seed_dict, identifier="merge")
+
+

{cobweb-launcher-1.1.12 → cobweb-launcher-1.1.14}/cobweb/launchers/launcher.py

@@ -93,7 +93,7 @@ class Launcher(threading.Thread):
             @launcher.request
             def request(seed: Seed) -> Union[Request, BaseItem]:
                 ...
-                return Request(seed.url, seed)
+                yield Request(seed.url, seed)
         """
         def decorator(func):
             self.__CUSTOM_FUNC__["request"] = func

{cobweb-launcher-1.1.12 → cobweb-launcher-1.1.14}/cobweb/launchers/launcher_pro.py

@@ -117,7 +117,7 @@ class LauncherPro(Launcher):
                 seed = self.__LAUNCHER_QUEUE__['done'].pop()
                 if not seed:
                     break
-                if seed.params.identifier == DealModel.fail:
+                if seed.identifier == DealModel.fail:
                     f_seeds.append(seed.to_string)
                 elif self._done_model == 1:
                     s_seeds.append(seed.to_string)

{cobweb-launcher-1.1.12 → cobweb-launcher-1.1.14/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.12
+Version: 1.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.12 → cobweb-launcher-1.1.14}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.12",
+    version="1.1.14",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.1.12/cobweb/crawlers/__init__.py

@@ -1,2 +0,0 @@
-from .base_crawler import Crawler
-from .file_crawler import FileCrawlerAir, FileCrawlerPro

cobweb-launcher-1.1.12/cobweb/crawlers/base_crawler.py

@@ -1,122 +0,0 @@
-import threading
-
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
-from cobweb.constant import DealModel, LogTemplate
-from cobweb.utils import download_log_info
-from cobweb import setting
-
-
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            upload_queue: Queue,
-            custom_func: Union[Mapping[str, Callable]],
-            launcher_queue: Union[Mapping[str, Queue]],
-    ):
-        super().__init__()
-
-        self.upload_queue = upload_queue
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.launcher_queue = launcher_queue
-
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.max_retries = setting.SPIDER_MAX_RETRIES
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        stream = True if setting.DOWNLOAD_MODEL else False
-        return Request(seed.url, seed, stream=stream, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        pass
-
-    def get(self) -> Seed:
-        return self.launcher_queue['todo'].pop()
-
-    def spider(self):
-        while True:
-            seed = self.get()
-
-            if not seed:
-                continue
-
-            elif seed.params.retry >= self.max_retries:
-                seed.params.identifier = DealModel.fail
-                self.launcher_queue['done'].push(seed)
-                continue
-
-            item = self.request(seed)
-
-            if isinstance(item, Request):
-
-                download_iterators = self.download(item)
-
-                if not isgenerator(download_iterators):
-                    raise TypeError("download function isn't a generator")
-
-                seed_detail_log_info = download_log_info(seed.to_dict)
-
-                try:
-                    for it in download_iterators:
-                        if isinstance(it, Response):
-                            response_detail_log_info = download_log_info(it.to_dict)
-                            logger.info(LogTemplate.download_info.format(
-                                detail=seed_detail_log_info, retry=item.seed.params.retry,
-                                priority=item.seed.params.priority,
-                                seed_version=item.seed.params.seed_version,
-                                identifier=item.seed.params.identifier,
-                                status=it.response, response=response_detail_log_info
-                            ))
-                            parse_iterators = self.parse(it)
-                            if not isgenerator(parse_iterators):
-                                raise TypeError("parse function isn't a generator")
-                            for upload_item in parse_iterators:
-                                if not isinstance(upload_item, BaseItem):
-                                    raise TypeError("upload_item isn't BaseItem subclass")
-                                self.upload_queue.push(upload_item)
-                        elif isinstance(it, BaseItem):
-                            self.upload_queue.push(it)
-                        elif isinstance(it, Seed):
-                            self.launcher_queue['new'].push(it)
-                        elif isinstance(it, str) and it == DealModel.poll:
-                            self.launcher_queue['todo'].push(item)
-                            break
-                        elif isinstance(it, str) and it == DealModel.done:
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        elif isinstance(it, str) and it == DealModel.fail:
-                            seed.params.identifier = DealModel.fail
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        else:
-                            raise TypeError("yield value type error!")
-
-                except Exception as e:
-                    logger.info(LogTemplate.download_exception.format(
-                        detail=seed_detail_log_info, retry=seed.params.retry,
-                        priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.params.identifier, exception=e
-                    ))
-                    seed.params.retry += 1
-                    self.launcher_queue['todo'].push(seed)
-
-            elif isinstance(item, BaseItem):
-                self.upload_queue.push(item)
-
-    def run(self):
-        for index in range(self.spider_thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
-

cobweb-launcher-1.1.12/cobweb/crawlers/file_crawler.py

@@ -1,173 +0,0 @@
-
-from typing import Union
-from cobweb import setting
-from cobweb.utils import OssUtil
-from cobweb.crawlers import Crawler
-from cobweb.base import Seed, BaseItem, Request, Response
-from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
-
-
-oss_util = OssUtil()
-
-
-class FileCrawlerAir(Crawler):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
-        try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
-            if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    response.close()
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
-
-class FileCrawlerPro(FileCrawlerAir):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
-        try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
-            if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")