cobweb-launcher 3.1.29__py3-none-any.whl → 3.1.31__py3-none-any.whl

cobweb/base/request.py

@@ -1,5 +1,7 @@
 import random
 import logging
+import time
+
 import requests
 
 from urllib.parse import urlparse
@@ -215,7 +217,7 @@ class FileTypeDetector:
             result['methods_used'].append('extension')
 
         # 3. 如果前两种方法不确定，使用文件签名检测
-        if result['confidence'] in ['unknown', 'medium']:
+        if data and result['confidence'] in ['unknown', 'medium']:
             signature_detected = self.detect_by_signature(data)
             if signature_detected:
                 if not result['detected_type']:
@@ -309,6 +311,8 @@ class Request:
         """
         self.scheme = None
         self.netloc = None
+        self.detector_info = None
+        self.content_length = None
         self._validate_url(url)
 
         self.url = url
@@ -408,6 +412,124 @@ class Request:
         """下载方法，为了向后兼容性保留"""
         return self.execute()
 
+    def normal_download(self, file_type_detect: bool = True) -> bytes:
+        """普通下载模式"""
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        response = requests.request(
+            method=self.method,
+            url=self.url,
+            **detect_settings
+        )
+
+        if self.check_status_code:
+            response.raise_for_status()
+
+        content_type = response.headers.get('content-type')
+        result = response.content
+        response.close()
+
+        if file_type_detect and not self.detector_info:
+            head_data = result[:64]
+            detector = FileTypeDetector()
+            self.detector_info = detector.get_detailed_info(
+                url=self.url, content_type=content_type, data=head_data
+            )
+
+        return result
+
+    def range_download(self, start: int = 0, chunk_size: int = 1024, file_type_detect: bool = True) -> iter(bytes):
+        # 分块下载
+        downloaded = start
+        retry_count = 0
+        max_retries = 3
+
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        if file_type_detect and not self.detector_info:
+            detect_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
+            test_response = requests.request(
+                method=self.method,
+                url=self.url,
+                **detect_settings
+            )
+            content_type = test_response.headers.get("Content-Type")
+            head_data = test_response.content
+            test_response.close()
+
+            detector = FileTypeDetector()
+            self.detector_info = detector.get_detailed_info(
+                url=self.url, content_type=content_type, data=head_data
+            )
+
+        while downloaded < self.content_length:
+            _start = downloaded
+            _end = min(downloaded + chunk_size - 1, self.total_size - 1)
+            detect_settings.setdefault("headers", {})['Range'] = f"bytes={_start}-{_end}"
+
+            response = requests.request(
+                method=self.method,
+                url=self.url,
+                **detect_settings
+            )
+
+            if response.status_code == 206:
+                chunk_data = response.content
+                yield chunk_data
+                downloaded += len(chunk_data)
+                retry_count = 0  # 重置重试计数
+                logging.info(f"下载进度: {downloaded}/{self.total_size} ({downloaded / self.total_size * 100:.1f}%)")
+
+            elif response.status_code == 416:  # Range Not Satisfiable
+                logging.info("Range请求超出范围")
+                break
+
+            else:
+                logging.debug(f"Range请求失败: {response.status_code}")
+                if retry_count < max_retries:
+                    retry_count += 1
+                    time.sleep(0.5)
+                    continue
+
+            response.close()
+
+    def detect_accept_ranges(self) -> bool:
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        head_response = requests.head(self.url, **detect_settings)
+        if head_response.status_code not in [200, 206]:
+            logging.error(f"HEAD请求失败: {head_response.status_code}")
+            raise ValueError("HTTP状态码错误")
+
+        self.content_length = int(head_response.headers.get('content-length', 0))
+        accept_ranges = str(head_response.headers.get('accept-ranges')).lower()
+
+        supports_range = True
+
+        # 根据检测结果使用不同下载方式
+        if accept_ranges == 'none' or not self.content_length:
+            supports_range = False
+        else:
+            test_range_settings = detect_settings.copy()
+            test_range_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
+            test_response = requests.request(
+                method=self.method,
+                url=self.url,
+                **test_range_settings
+            )
+            if test_response.status_code == 206:
+                actual_length = len(test_response.content)
+                if actual_length != 64:
+                    logging.debug(f"⚠️ Range请求返回长度不匹配: 期望64, 实际{actual_length}")
+                    supports_range = False
+            else:
+                supports_range = False
+
+        return supports_range
+
     def detect_file_type(self) -> Dict[str, Any]:
         """
         检测文件类型。
@@ -452,7 +574,7 @@ class Request:
 
     @property
     def to_dict(self) -> Dict[str, Any]:
-        excluded_keys = {"request_settings"}
+        excluded_keys = {"request_settings", "url", "seed", "method", "check_status_code"}
         result = {
             key: value for key, value in self.__dict__.items()
             if not key.startswith('_') and key not in excluded_keys

{cobweb_launcher-3.1.29.dist-info → cobweb_launcher-3.1.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.29
+Version: 3.1.31
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.29.dist-info → cobweb_launcher-3.1.31.dist-info}/RECORD

@@ -5,7 +5,7 @@ cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
 cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
 cobweb/base/item.py,sha256=1bS4U_3vzI2jzSSeoEbLoLT_5CfgLPopWiEYtaahbvw,1674
 cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
-cobweb/base/request.py,sha256=LBI2eCoXtQcUh6XQT813Q3rzhlhUFSm_UBGayYWIHKI,16052
+cobweb/base/request.py,sha256=WQq35IjRlUO_3bBBvgyuFb_9vyy7WjK-CJBkcYbEFwg,20597
 cobweb/base/response.py,sha256=L3sX2PskV744uz3BJ8xMuAoAfGCeh20w8h0Cnd9vLo0,11377
 cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
 cobweb/base/task_queue.py,sha256=2MqGpHGNmK5B-kqv7z420RWyihzB9zgDHJUiLsmtzOI,6402
@@ -34,8 +34,8 @@ cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,11
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.29.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.29.dist-info/METADATA,sha256=0v2KV50bVlPLBdWIp2DXQscDy_tWmKYL3lCtA7aiRoM,6051
-cobweb_launcher-3.1.29.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.29.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.29.dist-info/RECORD,,
+cobweb_launcher-3.1.31.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.31.dist-info/METADATA,sha256=m15hyIjwAYFJyAOKXrmTEuWlVPvAySayjWjNHwVO4CY,6051
+cobweb_launcher-3.1.31.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.31.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.31.dist-info/RECORD,,