cobweb-launcher 1.2.48__tar.gz → 1.2.50__tar.gz

{cobweb-launcher-1.2.48/cobweb_launcher.egg-info → cobweb-launcher-1.2.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.48
+Version: 1.2.50
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.48 → cobweb-launcher-1.2.50}/cobweb/base/seed.py

@@ -5,11 +5,13 @@ import hashlib
 
 class SeedParams:
 
-    def __init__(self, retry, priority, seed_version, seed_status=None):
+    def __init__(self, retry, priority, seed_version, seed_status=None, proxy_type=None, proxy=None):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
         self.seed_status = seed_status
+        self.proxy_type = proxy_type
+        self.proxy = proxy
 
 
 class Seed:
@@ -18,7 +20,9 @@ class Seed:
         "retry",
         "priority",
         "seed_version",
-        "seed_status"
+        "seed_status",
+        "proxy_type",
+        "proxy"
     ]
 
     def __init__(
@@ -29,6 +33,8 @@ class Seed:
             priority=None,
             seed_version=None,
             seed_status=None,
+            proxy_type=None,
+            proxy=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -51,6 +57,8 @@ class Seed:
             "priority": priority,
             "seed_version": seed_version,
             "seed_status": seed_status,
+            "proxy_type": proxy_type,
+            "proxy": proxy
         }
 
         if kwargs:

{cobweb-launcher-1.2.48 → cobweb-launcher-1.2.50}/cobweb/crawlers/crawler.py

@@ -19,7 +19,6 @@ from cobweb.base import (
     logger
 )
 from cobweb.utils import LoghubDot
-proxy_type = os.getenv("PROXY_TYPE", "")
 
 
 class Crawler(threading.Thread):
@@ -149,7 +148,10 @@ class Crawler(threading.Thread):
                                         topic=urlparse(download_item.response.request.url).netloc,
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
-                                        url=download_item.response.url, proxy_type=proxy_type,
+                                        url=download_item.response.url,
+                                        proxy_type=seed.params.proxy_type,
+                                        proxy=seed.params.proxy,
+                                        project=self.project, task=self.task,
                                     )
                                 parse_iterators = self.parse(download_item)
                                 if not isgenerator(parse_iterators):
@@ -177,7 +179,10 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
-                    proxy_type=proxy_type,
+                    proxy_type=seed.params.proxy_type,
+                    proxy=seed.params.proxy,
+                    project=self.project,
+                    task=self.task,
                     msg=exception_msg,
                 )
                 logger.info(LogTemplate.download_exception.format(

{cobweb-launcher-1.2.48 → cobweb-launcher-1.2.50}/cobweb/utils/dotting.py

@@ -24,16 +24,7 @@ class LoghubDot:
                 temp[key] = value
         contents = sorted(temp.items())
         log_item.set_contents(contents)
-        # log_items.append(log_item)
-        # request = PutLogsRequest(
-        #     project="databee-download-log",
-        #     logstore="log",
-        #     topic=topic,
-        #     logitems=log_items,
-        #     compress=True
-        # )
         self.queue.push((topic, log_item), direct_insertion=True)
-        # self.client.put_logs(request=request)
 
     def build_run(self):
         while True:

{cobweb-launcher-1.2.48 → cobweb-launcher-1.2.50/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.48
+Version: 1.2.50
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.48 → cobweb-launcher-1.2.50}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.48",
+    version="1.2.50",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",