cobweb-launcher 1.0.6__tar.gz → 1.0.8__tar.gz

{cobweb-launcher-1.0.6/cobweb_launcher.egg-info → cobweb-launcher-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.6
+Version: 1.0.8
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.0.6 → cobweb-launcher-1.0.8}/cobweb/crawlers/file_crawler.py

@@ -20,8 +20,6 @@ class CrawlerAir(Crawler):
             if CrawlerAir.oss_util.exists(key):
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             end = seed_dict.get("end", "")
             start = seed_dict.get("start", "0")
@@ -37,20 +35,20 @@ class CrawlerAir(Crawler):
 
                 response = item.download()
 
-                content_length = response.headers.get("content-length") or 0
+                content_length = int(response.headers.get("content-length", 0))
                 content_type = response.headers.get("content-type", "").split(";")[0]
                 if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
                     yield Response(
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
@@ -80,15 +78,11 @@ class CrawlerAir(Crawler):
                         content_length += len(content)
                     CrawlerAir.oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             elif item.seed.params.identifier == "merge":
                 CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
         except OssDBPutPartError:
             yield Seed(seed_dict)
         except OssDBMergeError:
@@ -123,20 +117,20 @@ class CrawlerPro(Crawler):
 
                 response = item.download()
 
-                content_length = response.headers.get("content-length") or 0
+                content_length = int(response.headers.get("content-length", 0))
                 content_type = response.headers.get("content-type", "").split(";")[0]
                 if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
                     yield Response(
                         item.seed, response, filter=True, msg=f"response content type is {content_type}",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                elif position == 1 and min_upload_size >= content_length > 0:
                     """过小文件标识返回"""
                     yield Response(
                         item.seed, response, filter=True, msg="file size is too small",
                         bucket_name=bucket_name, data_size=content_length, **seed_dict
                     )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                elif position == 1 and chunk_size > content_length > min_upload_size:
                     """小文件直接下载"""
                     for part_data in response.iter_content(chunk_size):
                         content += part_data
@@ -166,15 +160,12 @@ class CrawlerPro(Crawler):
                         content_length += len(content)
                     CrawlerAir.oss_util.merge(key, upload_id)
                     yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
 
             elif item.seed.params.identifier == "merge":
                 CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
                 content_length = CrawlerAir.oss_util.head(key).content_length
                 yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+
         except OssDBPutPartError:
             yield Seed(seed_dict)
         except OssDBMergeError:

{cobweb-launcher-1.0.6 → cobweb-launcher-1.0.8}/cobweb/setting.py

@@ -23,7 +23,7 @@ OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
 OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
-OSS_MIN_UPLOAD_SIZE = 1024 * 100
+OSS_MIN_UPLOAD_SIZE = 1024
 OSS_CHUNK_SIZE = 1024 ** 2
 
 # 采集器选择

{cobweb-launcher-1.0.6 → cobweb-launcher-1.0.8}/cobweb/utils/tools.py

@@ -39,4 +39,4 @@ def dynamic_load_class(model_info):
 
 
 def download_log_info(item:dict) -> str:
-    return "\n".join([" " * 12 + f"{k.ljust(14)}:    {v}" for k, v in item.items()])
+    return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

{cobweb-launcher-1.0.6 → cobweb-launcher-1.0.8/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.6
+Version: 1.0.8
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.0.6 → cobweb-launcher-1.0.8}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.0.6",
+    version="1.0.8",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",