cobweb-launcher 1.1.11__tar.gz → 1.1.13__tar.gz

{cobweb-launcher-1.1.11/cobweb_launcher.egg-info → cobweb-launcher-1.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.11
+Version: 1.1.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.11 → cobweb-launcher-1.1.13}/cobweb/base/response.py

@@ -20,3 +20,4 @@ class Response:
         _dict.pop('seed')
         _dict.pop('response')
         return _dict
+

{cobweb-launcher-1.1.11 → cobweb-launcher-1.1.13}/cobweb/base/seed.py

@@ -5,11 +5,10 @@ import hashlib
 
 class SeedParams:
 
-    def __init__(self, retry, priority, seed_version, identifier=None):
+    def __init__(self, retry, priority, seed_version):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
-        self.identifier = identifier
 
 
 class Seed:
@@ -18,7 +17,6 @@ class Seed:
         "retry",
         "priority",
         "seed_version",
-        "identifier"
     ]
 
     def __init__(
@@ -28,7 +26,6 @@ class Seed:
             retry=None,
             priority=None,
             seed_version=None,
-            identifier=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -50,7 +47,6 @@ class Seed:
             "retry": retry,
             "priority": priority,
             "seed_version": seed_version,
-            "identifier": identifier
         }
 
         if kwargs:

cobweb-launcher-1.1.13/cobweb/crawlers/__init__.py

@@ -0,0 +1,2 @@
+from .base_crawler import Crawler
+from .file_crawler import FileCrawlerAir

{cobweb-launcher-1.1.11 → cobweb-launcher-1.1.13}/cobweb/crawlers/base_crawler.py

@@ -77,7 +77,7 @@ class Crawler(threading.Thread):
                                 detail=seed_detail_log_info, retry=item.seed.params.retry,
                                 priority=item.seed.params.priority,
                                 seed_version=item.seed.params.seed_version,
-                                identifier=item.seed.params.identifier,
+                                identifier=item.seed.identifier or "",
                                 status=it.response, response=response_detail_log_info
                             ))
                             parse_iterators = self.parse(it)
@@ -108,7 +108,7 @@ class Crawler(threading.Thread):
                     logger.info(LogTemplate.download_exception.format(
                         detail=seed_detail_log_info, retry=seed.params.retry,
                         priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.params.identifier, exception=e
+                        identifier=seed.identifier or "", exception=e
                     ))
                     seed.params.retry += 1
                     self.launcher_queue['todo'].push(seed)

cobweb-launcher-1.1.13/cobweb/crawlers/file_crawler.py

@@ -0,0 +1,98 @@
+import os
+from typing import Union
+from cobweb import setting
+from cobweb.utils import OssUtil
+from cobweb.crawlers import Crawler
+from cobweb.base import Seed, BaseItem, Request, Response
+from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
+
+
+oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
+
+
+class FileCrawlerAir(Crawler):
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        seed_dict = item.seed.to_dict
+        seed_dict["bucket_name"] = oss_util.bucket
+        try:
+            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
+
+            if oss_util.exists(key):
+                seed_dict["data_size"] = oss_util.head(key).content_length
+                yield Response(item.seed, "exists", **seed_dict)
+
+            else:
+                seed_dict.setdefault("end", "")
+                seed_dict.setdefault("start", 0)
+
+                if seed_dict["end"] or seed_dict["start"]:
+                    start, end = seed_dict["start"], seed_dict["end"]
+                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+                if not item.seed.identifier:
+                    content = b""
+                    chunk_size = oss_util.chunk_size
+                    min_upload_size = oss_util.min_upload_size
+                    seed_dict.setdefault("position", 1)
+
+                    response = item.download()
+
+                    content_type = response.headers.get("content-type", "").split(";")[0]
+                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
+
+                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                        """过滤响应文件类型"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = f"response content type is {content_type}"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
+                        """过小文件标识返回"""
+                        response.close()
+                        seed_dict["filter"] = True
+                        seed_dict["msg"] = "file size is too small"
+                        yield Response(item.seed, response, **seed_dict)
+
+                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
+                        """小文件直接下载"""
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                        response.close()
+                        oss_util.put(key, content)
+                        yield Response(item.seed, response, **seed_dict)
+
+                    else:
+                        """中大文件同步分片下载"""
+                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
+
+                        for part_data in response.iter_content(chunk_size):
+                            content += part_data
+                            if len(content) >= chunk_size:
+                                upload_data = content[:chunk_size]
+                                content = content[chunk_size:]
+                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                                seed_dict['start'] += len(upload_data)
+                                seed_dict['position'] += 1
+
+                        response.close()
+
+                        if content:
+                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
+                        oss_util.merge(key, seed_dict["upload_id"])
+                        seed_dict["data_size"] = oss_util.head(key).content_length
+                        yield Response(item.seed, response, **seed_dict)
+
+                elif item.seed.identifier == "merge":
+                    oss_util.merge(key, seed_dict["upload_id"])
+                    seed_dict["data_size"] = oss_util.head(key).content_length
+                    yield Response(item.seed, "merge", **seed_dict)
+
+        except OssDBPutPartError:
+            yield Seed(seed_dict)
+        except OssDBMergeError:
+            yield Seed(seed_dict, identifier="merge")
+
+

{cobweb-launcher-1.1.11 → cobweb-launcher-1.1.13/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.11
+Version: 1.1.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.11 → cobweb-launcher-1.1.13}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.11",
+    version="1.1.13",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.1.11/cobweb/crawlers/__init__.py

@@ -1,2 +0,0 @@
-from .base_crawler import Crawler
-from .file_crawler import FileCrawlerAir, FileCrawlerPro

cobweb-launcher-1.1.11/cobweb/crawlers/file_crawler.py

@@ -1,173 +0,0 @@
-
-from typing import Union
-from cobweb import setting
-from cobweb.utils import OssUtil
-from cobweb.crawlers import Crawler
-from cobweb.base import Seed, BaseItem, Request, Response
-from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
-
-
-oss_util = OssUtil()
-
-
-class FileCrawlerAir(Crawler):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
-        try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
-            if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    response.close()
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
-
-class FileCrawlerPro(FileCrawlerAir):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = oss_util.bucket
-        try:
-            key = item.seed.oss_path or getattr(item, "oss_path", None)
-            if oss_util.exists(key):
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = oss_util.chunk_size
-                min_upload_size = oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = int(response.headers.get("content-length", 0))
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and min_upload_size >= content_length > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                    response.close()
-                elif position == 1 and chunk_size > content_length > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    if content:
-                        oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-
-            elif item.seed.params.identifier == "merge":
-                oss_util.merge(key, seed_dict["upload_id"])
-                content_length = oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")