cobweb-launcher 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

cobweb/crawlers/base_crawler.py

@@ -71,7 +71,7 @@ class Crawler(threading.Thread):
                 try:
                     for it in download_iterators:
                         if isinstance(it, Response):
-                            response_detail_log_info = download_log_info(it.to_dict())
+                            response_detail_log_info = download_log_info(it.to_dict)
                             logger.info(LogTemplate.download_info.format(
                                 detail=seed_detail_log_info, retry=item.seed.params.retry,
                                 priority=item.seed.params.priority,

cobweb/crawlers/file_crawler.py

@@ -20,8 +20,6 @@ class CrawlerAir(Crawler):
             if CrawlerAir.oss_util.exists(key):
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             end = seed_dict.get("end", "")
             start = seed_dict.get("start", "0")
@@ -80,15 +78,11 @@ class CrawlerAir(Crawler):
                         content_length += len(content)
                     CrawlerAir.oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             elif item.seed.params.identifier == "merge":
                 CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
         except OssDBPutPartError:
             yield Seed(seed_dict)
         except OssDBMergeError:

cobweb/setting.py

@@ -23,7 +23,7 @@ OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
 OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_MIN_UPLOAD_SIZE = 1024 * 100
+OSS_MIN_UPLOAD_SIZE = 1024
 OSS_CHUNK_SIZE = 1024 ** 2
 
 # 采集器选择

cobweb/utils/tools.py

@@ -39,4 +39,4 @@ def dynamic_load_class(model_info):
 
 
 def download_log_info(item:dict) -> str:
-    return "\n".join([" " * 12 + f"{k.ljust(14)}:    {v}" for k, v in item.items()])
+    return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

{cobweb_launcher-1.0.5.dist-info → cobweb_launcher-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.5
+Version: 1.0.7
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.0.5.dist-info → cobweb_launcher-1.0.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=IkGcdTU6fEBNyzWowcJSSMdErntFM1kmu4WUp1BgImU,45
 cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
-cobweb/setting.py,sha256=T693DAwLFLs9P6ZEvugP99UzXn-8PLeMEgdxRmL6cb4,1955
+cobweb/setting.py,sha256=fYK8KgrZYwSoJC9ywsopAtV0mdRw0_MGAoYORgET01M,1949
 cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -10,8 +10,8 @@ cobweb/base/request.py,sha256=b08AtUSZjlFLEFIEw5uGS__FjU6QSldL20-UjZD0LbI,2128
 cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
-cobweb/crawlers/base_crawler.py,sha256=-3WQSMrMROensTrTulh-eLK61qBsZDPoK2AM-sQwUHA,4893
-cobweb/crawlers/file_crawler.py,sha256=lOLsWxDKUQy0uJZ1NlXIPFiOJ17ufcveJIJD0Wqsqy8,9696
+cobweb/crawlers/base_crawler.py,sha256=snfjGYV9oZl3UnrxUcpAD9cK0n_rPeuyUYQkU1aCBrA,4891
+cobweb/crawlers/file_crawler.py,sha256=MyuJP0v4Rn4kFxCKb7JRInlzzwj8NsY0-jOHhpLFpTk,9085
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=XE1ebIi_4e0KBKiyPdKX7l2lSgV5vhMwEhNDlAcsccU,4247
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
@@ -24,9 +24,9 @@ cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zL
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
-cobweb/utils/tools.py,sha256=8oLxkzwaYcDqKXXuLI3A_lNabyLBr7HSPgTF6x4xbnk,1239
-cobweb_launcher-1.0.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.0.5.dist-info/METADATA,sha256=qZCm5HEq0VbbPTtZbWoRh6a_Jx7JsUtk_VIXU_-zQAA,1245
-cobweb_launcher-1.0.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.0.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.0.5.dist-info/RECORD,,
+cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
+cobweb_launcher-1.0.7.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.0.7.dist-info/METADATA,sha256=vsC5TBqJEwb85vKo0PXhH84sO8YYtZPPeqI1WGLpwNY,1245
+cobweb_launcher-1.0.7.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.0.7.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.0.7.dist-info/RECORD,,