webtoolkit 0.0.192__tar.gz → 0.0.194__tar.gz

{webtoolkit-0.0.192 → webtoolkit-0.0.194}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.0.192
+Version: 0.0.194
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: beautifulsoup4 (>=4.13.5,<5.0.0)
 Requires-Dist: brutefeedparser (>=0.10.5,<0.11.0)
 Requires-Dist: lxml (>=5.4.0,<6.0.0)

{webtoolkit-0.0.192 → webtoolkit-0.0.194}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.0.192"
+version = "0.0.194"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.0.192 → webtoolkit-0.0.194}/webtoolkit/baseurl.py

@@ -94,12 +94,18 @@ class BaseUrl(ContentInterface):
         request.crawler_name = "RequestsCrawler"
         request.crawler_type = RequestsCrawler(url)
 
+        if request.timeout_s is None or request.timeout_s == 0:
+            request.timeout_s = 60 * 5
+
         return request
 
     def get_request_for_request(self, request):
         request.crawler_name = "RequestsCrawler"
         request.crawler_type = RequestsCrawler(request.url)
 
+        if request.timeout_s is None or request.timeout_s == 0:
+            request.timeout_s = 60 * 5
+
         return request
 
     def get_handlers(self):

{webtoolkit-0.0.192 → webtoolkit-0.0.194}/webtoolkit/remoteserver.py

@@ -25,15 +25,16 @@ class RemoteServer(object):
     def __init__(self, remote_server=None, timeout_s=30):
         self.remote_server = remote_server
         if not self.remote_server:
-            CRAWLER_BUDDY_SERVER = os.environ.get("CRAWLER_BUDDY_SERVER")
-            CRAWLER_BUDDY_PORT = os.environ.get("CRAWLER_BUDDY_PORT")
-            if CRAWLER_BUDDY_SERVER and CRAWLER_BUDDY_PORT:
-                self.remote_server = (
-                    f"http://{CRAWLER_BUDDY_SERVER}:{CRAWLER_BUDDY_PORT}"
-                )
+            self.remote_server = RemoteServer.get_remote_server_location()
 
         self.timeout_s = timeout_s
 
+    def get_remote_server_location():
+        CRAWLER_BUDDY_SERVER = os.environ.get("CRAWLER_BUDDY_SERVER")
+        CRAWLER_BUDDY_PORT = os.environ.get("CRAWLER_BUDDY_PORT")
+        if CRAWLER_BUDDY_SERVER and CRAWLER_BUDDY_PORT:
+            return f"http://{CRAWLER_BUDDY_SERVER}:{CRAWLER_BUDDY_PORT}"
+
     def get_getj(self, request=None, url=None):
         """
         @returns None in case of error
@@ -145,6 +146,9 @@ class RemoteServer(object):
         """
         @param link_call Remote server endpoint
         @param url Url for which we call Remote server
+
+        Note: there should always be a timeout. Server might stop responding,
+        it could have hanged, etc.
         """
         url = request.url
 
@@ -154,8 +158,10 @@ class RemoteServer(object):
 
         text = None
 
-        timeout_s = 50
+        # it is hard to think of a good deafult value
+        timeout_s = 60
         if request.timeout_s is not None:
+            # remote server will have timeout_s we add some wiggle room for transmission
             timeout_s = request.timeout_s
             timeout_s += 5
 

{webtoolkit-0.0.192 → webtoolkit-0.0.194}/webtoolkit/remoteurl.py

@@ -43,8 +43,8 @@ class RemoteUrl(ContentInterface):
         """
         super().__init__(url=url, contents=None)
         self.request = request
-        self.remote_server_location = remote_server_location
-        self.server = RemoteServer(remote_server_location)
+        self.remote_server_location=remote_server_location
+        self.server = RemoteServer(remote_server=self.remote_server_location)
         self.all_properties = all_properties
         self.social_properties = social_properties
 
@@ -52,6 +52,9 @@ class RemoteUrl(ContentInterface):
         if self.all_properties:
             self.get_responses()
 
+    def get_remote_server_location():
+        return RemoteServer.get_remote_server_location()
+
     def get_responses(self):
         """Provides URL responses"""
         if self.all_properties is None: