cobweb-launcher 1.0.7__py3-none-any.whl → 1.0.8__py3-none-any.whl

cobweb/crawlers/file_crawler.py

@@ -35,20 +35,20 @@ class CrawlerAir(Crawler):
 
                 response = item.download()
 
-                content_length = response.headers.get("content-length") or 0
+                content_length = int(response.headers.get("content-length", 0))
                 content_type = response.headers.get("content-type", "").split(";")[0]
                 if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
                     yield Response(
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
@@ -117,20 +117,20 @@ class CrawlerPro(Crawler):
 
                 response = item.download()
 
-                content_length = response.headers.get("content-length") or 0
+                content_length = int(response.headers.get("content-length", 0))
                 content_type = response.headers.get("content-type", "").split(";")[0]
                 if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
                     yield Response(
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
@@ -160,15 +160,12 @@ class CrawlerPro(Crawler):
                         content_length += len(content)
                     CrawlerAir.oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             elif item.seed.params.identifier == "merge":
                 CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+
         except OssDBPutPartError:
             yield Seed(seed_dict)
         except OssDBMergeError:

{cobweb_launcher-1.0.7.dist-info → cobweb_launcher-1.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.7
+Version: 1.0.8
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.0.7.dist-info → cobweb_launcher-1.0.8.dist-info}/RECORD

@@ -11,7 +11,7 @@ cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
 cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
 cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
 cobweb/crawlers/base_crawler.py,sha256=snfjGYV9oZl3UnrxUcpAD9cK0n_rPeuyUYQkU1aCBrA,4891
-cobweb/crawlers/file_crawler.py,sha256=MyuJP0v4Rn4kFxCKb7JRInlzzwj8NsY0-jOHhpLFpTk,9085
+cobweb/crawlers/file_crawler.py,sha256=AeK22bakMbwDfqrITlX6ssvR2KSpHQ3s8YEuzX_Bluk,8670
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=XE1ebIi_4e0KBKiyPdKX7l2lSgV5vhMwEhNDlAcsccU,4247
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
@@ -25,8 +25,8 @@ cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzX
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
 cobweb/utils/tools.py,sha256=bVd3iRGBvwhohQAH7AXTTjbmQ54Z35K0O-fatEyhePU,1249
-cobweb_launcher-1.0.7.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.0.7.dist-info/METADATA,sha256=vsC5TBqJEwb85vKo0PXhH84sO8YYtZPPeqI1WGLpwNY,1245
-cobweb_launcher-1.0.7.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.0.7.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.0.7.dist-info/RECORD,,
+cobweb_launcher-1.0.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.0.8.dist-info/METADATA,sha256=XxTFgYdij-0ct7QV4cqz8Tg4b7JhoOWQ3j_K1_SplEc,1245
+cobweb_launcher-1.0.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.0.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.0.8.dist-info/RECORD,,