cobweb-launcher 1.1.20__tar.gz → 1.1.22__tar.gz

{cobweb-launcher-1.1.20/cobweb_launcher.egg-info → cobweb-launcher-1.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.20
+Version: 1.1.22
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.20 → cobweb-launcher-1.1.22}/cobweb/crawlers/base_crawler.py

@@ -1,4 +1,5 @@
 import threading
+import time
 
 from inspect import isgenerator
 from typing import Union, Callable, Mapping
@@ -132,6 +133,8 @@ class Crawler(threading.Thread):
                 ))
                 seed.params.retry += 1
                 self.launcher_queue['todo'].push(seed)
+            finally:
+                time.sleep(0.1)
 
     def run(self):
         for index in range(self.spider_thread_num):

{cobweb-launcher-1.1.20 → cobweb-launcher-1.1.22}/cobweb/launchers/launcher_pro.py

@@ -125,12 +125,13 @@ class LauncherPro(Launcher):
                     seeds.append(seed.to_string)
             if seeds:
                 self._db.zrem(self._todo, *seeds)
+                self._remove_doing_seeds(seeds)
             if s_seeds:
                 self._db.done([self._todo, self._done], *s_seeds)
+                self._remove_doing_seeds(s_seeds)
             if f_seeds:
                 self._db.done([self._todo, self._fail], *f_seeds)
-
-            self._remove_doing_seeds(seeds)
+                self._remove_doing_seeds(f_seeds)
 
             if status:
                 time.sleep(self._done_queue_wait_seconds)

{cobweb-launcher-1.1.20 → cobweb-launcher-1.1.22}/cobweb/utils/oss.py

@@ -1,6 +1,7 @@
+from typing import List
 from cobweb import setting
 from requests import Response
-from oss2 import Auth, Bucket, models
+from oss2 import Auth, Bucket, models, PartIterator
 from cobweb.exceptions import oss_db_exception
 from cobweb.base.decorators import decorator_oss_db
 
@@ -54,6 +55,10 @@ class OssUtil:
         """分片上传"""
         return self._client.upload_part(key, upload_id, position, data)
 
+    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+        """获取分片列表"""
+        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+
     @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
         """合并分片"""

{cobweb-launcher-1.1.20 → cobweb-launcher-1.1.22/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.20
+Version: 1.1.22
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.1.20 → cobweb-launcher-1.1.22}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.1.20",
+    version="1.1.22",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",