cobweb-launcher 3.1.29__tar.gz → 3.1.31__tar.gz

{cobweb-launcher-3.1.29 → cobweb-launcher-3.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.29
+Version: 3.1.31
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.29 → cobweb-launcher-3.1.31}/cobweb/base/request.py

@@ -1,5 +1,7 @@
 import random
 import logging
+import time
+
 import requests
 
 from urllib.parse import urlparse
@@ -215,7 +217,7 @@ class FileTypeDetector:
             result['methods_used'].append('extension')
 
         # 3. 如果前两种方法不确定，使用文件签名检测
-        if result['confidence'] in ['unknown', 'medium']:
+        if data and result['confidence'] in ['unknown', 'medium']:
             signature_detected = self.detect_by_signature(data)
             if signature_detected:
                 if not result['detected_type']:
@@ -309,6 +311,8 @@ class Request:
         """
         self.scheme = None
         self.netloc = None
+        self.detector_info = None
+        self.content_length = None
         self._validate_url(url)
 
         self.url = url
@@ -408,6 +412,124 @@ class Request:
         """下载方法，为了向后兼容性保留"""
         return self.execute()
 
+    def normal_download(self, file_type_detect: bool = True) -> bytes:
+        """普通下载模式"""
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        response = requests.request(
+            method=self.method,
+            url=self.url,
+            **detect_settings
+        )
+
+        if self.check_status_code:
+            response.raise_for_status()
+
+        content_type = response.headers.get('content-type')
+        result = response.content
+        response.close()
+
+        if file_type_detect and not self.detector_info:
+            head_data = result[:64]
+            detector = FileTypeDetector()
+            self.detector_info = detector.get_detailed_info(
+                url=self.url, content_type=content_type, data=head_data
+            )
+
+        return result
+
+    def range_download(self, start: int = 0, chunk_size: int = 1024, file_type_detect: bool = True) -> iter(bytes):
+        # 分块下载
+        downloaded = start
+        retry_count = 0
+        max_retries = 3
+
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        if file_type_detect and not self.detector_info:
+            detect_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
+            test_response = requests.request(
+                method=self.method,
+                url=self.url,
+                **detect_settings
+            )
+            content_type = test_response.headers.get("Content-Type")
+            head_data = test_response.content
+            test_response.close()
+
+            detector = FileTypeDetector()
+            self.detector_info = detector.get_detailed_info(
+                url=self.url, content_type=content_type, data=head_data
+            )
+
+        while downloaded < self.content_length:
+            _start = downloaded
+            _end = min(downloaded + chunk_size - 1, self.total_size - 1)
+            detect_settings.setdefault("headers", {})['Range'] = f"bytes={_start}-{_end}"
+
+            response = requests.request(
+                method=self.method,
+                url=self.url,
+                **detect_settings
+            )
+
+            if response.status_code == 206:
+                chunk_data = response.content
+                yield chunk_data
+                downloaded += len(chunk_data)
+                retry_count = 0  # 重置重试计数
+                logging.info(f"下载进度: {downloaded}/{self.total_size} ({downloaded / self.total_size * 100:.1f}%)")
+
+            elif response.status_code == 416:  # Range Not Satisfiable
+                logging.info("Range请求超出范围")
+                break
+
+            else:
+                logging.debug(f"Range请求失败: {response.status_code}")
+                if retry_count < max_retries:
+                    retry_count += 1
+                    time.sleep(0.5)
+                    continue
+
+            response.close()
+
+    def detect_accept_ranges(self) -> bool:
+        detect_settings = self.request_settings.copy()
+        detect_settings.pop('stream', None)
+
+        head_response = requests.head(self.url, **detect_settings)
+        if head_response.status_code not in [200, 206]:
+            logging.error(f"HEAD请求失败: {head_response.status_code}")
+            raise ValueError("HTTP状态码错误")
+
+        self.content_length = int(head_response.headers.get('content-length', 0))
+        accept_ranges = str(head_response.headers.get('accept-ranges')).lower()
+
+        supports_range = True
+
+        # 根据检测结果使用不同下载方式
+        if accept_ranges == 'none' or not self.content_length:
+            supports_range = False
+        else:
+            test_range_settings = detect_settings.copy()
+            test_range_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
+            test_response = requests.request(
+                method=self.method,
+                url=self.url,
+                **test_range_settings
+            )
+            if test_response.status_code == 206:
+                actual_length = len(test_response.content)
+                if actual_length != 64:
+                    logging.debug(f"⚠️ Range请求返回长度不匹配: 期望64, 实际{actual_length}")
+                    supports_range = False
+            else:
+                supports_range = False
+
+        return supports_range
+
     def detect_file_type(self) -> Dict[str, Any]:
         """
         检测文件类型。
@@ -452,7 +574,7 @@ class Request:
 
     @property
     def to_dict(self) -> Dict[str, Any]:
-        excluded_keys = {"request_settings"}
+        excluded_keys = {"request_settings", "url", "seed", "method", "check_status_code"}
         result = {
             key: value for key, value in self.__dict__.items()
             if not key.startswith('_') and key not in excluded_keys

{cobweb-launcher-3.1.29 → cobweb-launcher-3.1.31}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.29
+Version: 3.1.31
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-3.1.29 → cobweb-launcher-3.1.31}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="3.1.29",
+    version="3.1.31",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",