cobweb-launcher 1.1.21__py3-none-any.whl → 1.1.23__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -1,5 +1,6 @@
 import threading
 import time
+import traceback
 
 from inspect import isgenerator
 from typing import Union, Callable, Mapping
@@ -129,7 +130,8 @@ class Crawler(threading.Thread):
                     retry=seed.params.retry,
                     priority=seed.params.priority,
                     seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "", exception=e
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
                 ))
                 seed.params.retry += 1
                 self.launcher_queue['todo'].push(seed)

cobweb/utils/oss.py

@@ -1,6 +1,7 @@
+from typing import List
 from cobweb import setting
 from requests import Response
-from oss2 import Auth, Bucket, models
+from oss2 import Auth, Bucket, models, PartIterator
 from cobweb.exceptions import oss_db_exception
 from cobweb.base.decorators import decorator_oss_db
 
@@ -54,6 +55,10 @@ class OssUtil:
         """分片上传"""
         return self._client.upload_part(key, upload_id, position, data)
 
+    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+        """获取分片列表"""
+        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+
     @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
         """合并分片"""

{cobweb_launcher-1.1.21.dist-info → cobweb_launcher-1.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.1.21
+Version: 1.1.23
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.1.21.dist-info → cobweb_launcher-1.1.23.dist-info}/RECORD

@@ -10,7 +10,7 @@ cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=Rr3DTjD-abMA1_FYcQJZYNvQvcWMuEVcsIU6duqHrw4,75
-cobweb/crawlers/base_crawler.py,sha256=05uIIPs_1RdRKPfEcS8zAwgXbBODOJqAJUjWsMw7jQo,5651
+cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
@@ -23,10 +23,10 @@ cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
-cobweb/utils/oss.py,sha256=qAl05ybL2Jp6KFjHDHVMfmeBHQmDKPtZleHjHAY7LZc,3277
+cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.1.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.1.21.dist-info/METADATA,sha256=jPQSJUuw_07x61hjH52AhxralQQjb8yDVnR0ajtccmU,1246
-cobweb_launcher-1.1.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.1.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.1.21.dist-info/RECORD,,
+cobweb_launcher-1.1.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.1.23.dist-info/METADATA,sha256=X93ltTYRGmvdT8oWACxycFetJ2KV3BgZdBP_MtDkE5U,1246
+cobweb_launcher-1.1.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.1.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.1.23.dist-info/RECORD,,