cobweb-launcher 1.1.12__py3-none-any.whl → 1.1.13__py3-none-any.whl

cobweb/crawlers/__init__.py

@@ -1,2 +1,2 @@
 from .base_crawler import Crawler
-from .file_crawler import FileCrawlerAir, FileCrawlerPro
+from .file_crawler import FileCrawlerAir

cobweb/crawlers/base_crawler.py

@@ -58,18 +58,18 @@ class Crawler(threading.Thread):
                 self.launcher_queue['done'].push(seed)
                 continue
 
-            seed_detail_log_info = download_log_info(seed.to_dict)
+            item = self.request(seed)
 
-            try:
-                item = self.request(seed)
+            if isinstance(item, Request):
 
-                if isinstance(item, Request):
+                download_iterators = self.download(item)
 
-                    download_iterators = self.download(item)
+                if not isgenerator(download_iterators):
+                    raise TypeError("download function isn't a generator")
 
-                    if not isgenerator(download_iterators):
-                        raise TypeError("download function isn't a generator")
+                seed_detail_log_info = download_log_info(seed.to_dict)
 
+                try:
                     for it in download_iterators:
                         if isinstance(it, Response):
                             response_detail_log_info = download_log_info(it.to_dict)
@@ -77,7 +77,7 @@ class Crawler(threading.Thread):
                                 detail=seed_detail_log_info, retry=item.seed.params.retry,
                                 priority=item.seed.params.priority,
                                 seed_version=item.seed.params.seed_version,
-                                identifier=item.seed.params.identifier,
+                                identifier=item.seed.identifier or "",
                                 status=it.response, response=response_detail_log_info
                             ))
                             parse_iterators = self.parse(it)
@@ -104,22 +104,17 @@ class Crawler(threading.Thread):
                         else:
                             raise TypeError("yield value type error!")
 
-                elif isinstance(item, BaseItem):
-                    self.upload_queue.push(item)
-                else:
-                    raise TypeError(
-                        f"request func return value type error!"
-                        f"item.__class__ is  {item.__class__.__name__}"
-                    )
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info, retry=seed.params.retry,
-                    priority=seed.params.priority, seed_version=seed.params.seed_version,
-                    identifier=seed.params.identifier, exception=e
-                ))
-                seed.params.retry += 1
-                self.launcher_queue['todo'].push(seed)
-
+                except Exception as e:
+                    logger.info(LogTemplate.download_exception.format(
+                        detail=seed_detail_log_info, retry=seed.params.retry,
+                        priority=seed.params.priority, seed_version=seed.params.seed_version,
+                        identifier=seed.identifier or "", exception=e
+                    ))
+                    seed.params.retry += 1
+                    self.launcher_queue['todo'].push(seed)
+
+            elif isinstance(item, BaseItem):
+                self.upload_queue.push(item)
 
     def run(self):
         for index in range(self.spider_thread_num):

cobweb/crawlers/file_crawler.py

@@ -1,4 +1,4 @@
-
+import os
 from typing import Union
 from cobweb import setting
 from cobweb.utils import OssUtil
@@ -7,7 +7,7 @@ from cobweb.base import Seed, BaseItem, Request, Response
 from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
 
 
-oss_util = OssUtil()
+oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
 
 
 class FileCrawlerAir(Crawler):
@@ -15,159 +15,84 @@ class FileCrawlerAir(Crawler):
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
         seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
+        seed_dict["bucket_name"] = oss_util.bucket
         try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
-            if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    response.close()
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
+            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
 
-class FileCrawlerPro(FileCrawlerAir):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
-        try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
             if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                seed_dict["data_size"] = oss_util.head(key).content_length
+                yield Response(item.seed, "exists", **seed_dict)
+
+            else:
+                seed_dict.setdefault("end", "")
+                seed_dict.setdefault("start", 0)
+
+                if seed_dict["end"] or seed_dict["start"]:
+                    start, end = seed_dict["start"], seed_dict["end"]
+                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+                if not item.seed.identifier:
+                    content = b""
+                    chunk_size = oss_util.chunk_size
+                    min_upload_size = oss_util.min_upload_size
+                    seed_dict.setdefault("position", 1)
+
+                    response = item.download()
+
+                    content_type = response.headers.get("content-type", "").split(";")[0]
+                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
+
+                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                        """过滤响应文件类型"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = f"response content type is {content_type}"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
+                        """过小文件标识返回"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = "file size is too small"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
+                        """小文件直接下载"""
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                        response.close()
+                        oss_util.put(key, content)
+                        yield Response(item.seed, response, **seed_dict)
+
+                    else:
+                        """中大文件同步分片下载"""
+                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
+
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                            if len(content) >= chunk_size:
+                                upload_data = content[:chunk_size]
+                                content = content[chunk_size:]
+                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                                seed_dict['start'] += len(upload_data)
+                                seed_dict['position'] += 1
+
+                        response.close()
+
+                        if content:
+                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                        oss_util.merge(key, seed_dict["upload_id"])
+                        seed_dict["data_size"] = oss_util.head(key).content_length
+                        yield Response(item.seed, response, **seed_dict)
+
+                elif item.seed.identifier == "merge":
+                    oss_util.merge(key, seed_dict["upload_id"])
+                    seed_dict["data_size"] = oss_util.head(key).content_length
+                    yield Response(item.seed, "merge", **seed_dict)
 
         except OssDBPutPartError:
             yield Seed(seed_dict)
         except OssDBMergeError:
             yield Seed(seed_dict, identifier="merge")
+
+

{cobweb_launcher-1.1.12.dist-info → cobweb_launcher-1.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.12
+Version: 1.1.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.12.dist-info → cobweb_launcher-1.1.13.dist-info}/RECORD

@@ -9,9 +9,9 @@ cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
 cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=QxlXztWjV8VvcHu-cTzyoDHaDvoX26iyoJKiWk64HnE,2759
-cobweb/crawlers/__init__.py,sha256=_HAXBg7Sq8fsDGSjDm3AQz9aQtLZONpt5b8dSe607mI,91
-cobweb/crawlers/base_crawler.py,sha256=uR1wQ2sJpFovNoAK52293rF03O-jNbv24P5QoNt1tW0,5169
-cobweb/crawlers/file_crawler.py,sha256=AuKu2ZoDoccsI2COnsUWGxKQf67Y-87lL7xqiPv0LRI,8532
+cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
+cobweb/crawlers/base_crawler.py,sha256=H-cUfrn79PAI8Z7Xv_NhW-9J0G-SiBU4wtzymzAJMYc,4961
+cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.12.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.12.dist-info/METADATA,sha256=On83pLSNRDRqXO5NRz7ZUq3_Bq2pfFQfaSoTeNeSdMw,1246
-cobweb_launcher-1.1.12.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.12.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.12.dist-info/RECORD,,
+cobweb_launcher-1.1.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.13.dist-info/METADATA,sha256=ShOTXcWNMBOaLR50TU0QFjR25Ae9aZ-PNyBgwTkxckE,1246
+cobweb_launcher-1.1.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.13.dist-info/RECORD,,