cobweb-launcher 3.1.32__tar.gz → 3.1.33__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/PKG-INFO +1 -1
  2. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/request.py +33 -31
  3. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb_launcher.egg-info/PKG-INFO +1 -1
  4. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/setup.py +1 -1
  5. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/LICENSE +0 -0
  6. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/README.md +0 -0
  7. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/__init__.py +0 -0
  8. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/__init__.py +0 -0
  9. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/common_queue.py +0 -0
  10. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/item.py +0 -0
  11. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/logger.py +0 -0
  12. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/response.py +0 -0
  13. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/seed.py +0 -0
  14. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/task_queue.py +0 -0
  15. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/base/test.py +0 -0
  16. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/constant.py +0 -0
  17. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/crawlers/__init__.py +0 -0
  18. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/crawlers/crawler.py +0 -0
  19. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/db/__init__.py +0 -0
  20. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/db/api_db.py +0 -0
  21. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/db/redis_db.py +0 -0
  22. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/exceptions/__init__.py +0 -0
  23. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/exceptions/oss_db_exception.py +0 -0
  24. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/launchers/__init__.py +0 -0
  25. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/launchers/distributor.py +0 -0
  26. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/launchers/launcher.py +0 -0
  27. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/launchers/uploader.py +0 -0
  28. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/pipelines/__init__.py +0 -0
  29. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/pipelines/pipeline.py +0 -0
  30. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/pipelines/pipeline_csv.py +0 -0
  31. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/pipelines/pipeline_loghub.py +0 -0
  32. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/schedulers/__init__.py +0 -0
  33. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/schedulers/scheduler.py +0 -0
  34. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/schedulers/scheduler_with_redis.py +0 -0
  35. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/setting.py +0 -0
  36. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/__init__.py +0 -0
  37. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/bloom.py +0 -0
  38. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/decorators.py +0 -0
  39. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/dotting.py +0 -0
  40. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/oss.py +0 -0
  41. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb/utils/tools.py +0 -0
  42. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb_launcher.egg-info/SOURCES.txt +0 -0
  43. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb_launcher.egg-info/dependency_links.txt +0 -0
  44. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb_launcher.egg-info/requires.txt +0 -0
  45. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/cobweb_launcher.egg-info/top_level.txt +0 -0
  46. {cobweb-launcher-3.1.32 → cobweb-launcher-3.1.33}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.32
3
+ Version: 3.1.33
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
@@ -311,6 +311,7 @@ class Request:
311
311
  """
312
312
  self.scheme = None
313
313
  self.netloc = None
314
+ self.response = None
314
315
  self.detector_info = None
315
316
  self.content_length = None
316
317
  self._validate_url(url)
@@ -469,31 +470,32 @@ class Request:
469
470
  _end = min(downloaded + chunk_size - 1, self.total_size - 1)
470
471
  detect_settings.setdefault("headers", {})['Range'] = f"bytes={_start}-{_end}"
471
472
 
472
- response = requests.request(
473
+ self.response = requests.request(
473
474
  method=self.method,
474
475
  url=self.url,
475
476
  **detect_settings
476
477
  )
477
478
 
478
- if response.status_code == 206:
479
- chunk_data = response.content
479
+ if self.response.status_code == 206:
480
+ chunk_data = self.response.content
480
481
  yield chunk_data
481
482
  downloaded += len(chunk_data)
482
483
  retry_count = 0 # 重置重试计数
483
484
  logging.info(f"下载进度: {downloaded}/{self.total_size} ({downloaded / self.total_size * 100:.1f}%)")
484
485
 
485
- elif response.status_code == 416: # Range Not Satisfiable
486
+ elif self.response.status_code == 416: # Range Not Satisfiable
486
487
  logging.info("Range请求超出范围")
487
488
  break
488
489
 
489
490
  else:
490
- logging.debug(f"Range请求失败: {response.status_code}")
491
+ logging.debug(f"Range请求失败: {self.response.status_code}")
491
492
  if retry_count < max_retries:
492
493
  retry_count += 1
493
494
  time.sleep(0.5)
494
495
  continue
495
496
 
496
- response.close()
497
+ self.response.close()
498
+ self.response = None
497
499
 
498
500
  def detect_accept_ranges(self) -> bool:
499
501
  detect_settings = self.request_settings.copy()
@@ -505,38 +507,38 @@ class Request:
505
507
  raise ValueError("HTTP状态码错误")
506
508
 
507
509
  self.content_length = int(head_response.headers.get('content-length', 0))
508
- accept_ranges = str(head_response.headers.get('accept-ranges')).lower()
509
-
510
+ # accept_ranges = str(head_response.headers.get('accept-ranges')).lower()
510
511
  supports_range = True
511
512
 
512
513
  # 根据检测结果使用不同下载方式
513
- if accept_ranges == 'none' or not self.content_length:
514
- supports_range = False
515
- else:
516
- test_range_settings = detect_settings.copy()
517
- test_range_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
518
- test_response = requests.request(
519
- method=self.method,
520
- url=self.url,
521
- **test_range_settings
522
- )
523
- head_data = test_response.content
524
- content_type = test_response.headers.get("Content-Type")
514
+ # if accept_ranges == 'none' or not self.content_length:
515
+ # supports_range = False
516
+ # else:
517
+ test_range_settings = detect_settings.copy()
518
+ test_range_settings.setdefault("headers", {})['Range'] = "bytes=0-63"
519
+ test_response = requests.request(
520
+ method=self.method,
521
+ url=self.url,
522
+ **test_range_settings
523
+ )
524
+ head_data = test_response.content
525
+ content_type = test_response.headers.get("Content-Type")
525
526
 
526
- if test_response.status_code == 206:
527
- if len(head_data) != 64:
528
- logging.debug(f"⚠️ Range请求返回长度不匹配: 期望64, 实际{len(head_data)}")
529
- supports_range = False
530
- else:
531
- detector = FileTypeDetector()
532
- self.detector_info = detector.get_detailed_info(
533
- url=self.url, content_type=content_type, data=head_data
534
- )
535
- else:
527
+ if test_response.status_code == 206:
528
+ if len(head_data) != 64:
536
529
  supports_range = False
530
+ self.response = test_response
531
+ logging.debug(f"⚠️ Range请求返回长度不匹配: 期望64, 实际{len(head_data)}")
532
+ head_data = head_data[:64]
537
533
 
538
- test_response.close()
534
+ detector = FileTypeDetector()
535
+ self.detector_info = detector.get_detailed_info(
536
+ url=self.url, content_type=content_type, data=head_data
537
+ )
538
+ else:
539
+ supports_range = False
539
540
 
541
+ test_response.close()
540
542
  return supports_range
541
543
 
542
544
  def detect_file_type(self) -> Dict[str, Any]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cobweb-launcher
3
- Version: 3.1.32
3
+ Version: 3.1.33
4
4
  Summary: spider_hole
5
5
  Home-page: https://github.com/Juannie-PP/cobweb
6
6
  Author: Juannie-PP
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
5
5
 
6
6
  setup(
7
7
  name="cobweb-launcher",
8
- version="3.1.32",
8
+ version="3.1.33",
9
9
  packages=find_packages(),
10
10
  url="https://github.com/Juannie-PP/cobweb",
11
11
  license="MIT",