webtoolkit 0.1.67__tar.gz → 0.1.70__tar.gz

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.1.67
+Version: 0.1.70
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.1.67"
+version = "0.1.70"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/webtoolkit/remoteserver.py

@@ -192,7 +192,7 @@ class RemoteServer(object):
             print(str(E))
             return
 
-        if "success" in json_obj and not json_obj["success"]:
+        if json_obj and "success" in json_obj and not json_obj["success"]:
             print("Url:{} Remote error. Not a success".format(link_call))
             return
         return json_obj

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/webtoolkit/remoteurl.py

@@ -47,8 +47,6 @@ class RemoteUrl(ContentInterface):
         if request is not None and url is None:
             url = request.url
 
-        super().__init__(url=url, contents=None)
-
         self.request = request
         self.remote_server_location = remote_server_location
         self.server = RemoteServer(remote_server=self.remote_server_location, key=key, client_id=client_id)
@@ -56,6 +54,14 @@ class RemoteUrl(ContentInterface):
         self.social_properties = social_properties
 
         self.responses = None
+
+        if url is None and self.all_properties:
+            properties = self.get_properties()
+            if properties:
+                url = properties.get("link")
+
+        super().__init__(url=url, contents=None)
+
         if self.all_properties:
             self.get_responses()
 
@@ -77,12 +83,15 @@ class RemoteUrl(ContentInterface):
 
         return self.responses
 
-    def get_response(self):
+    def get_response(self, stream=None):
         """Provides URL response"""
         responses = self.get_responses()
         if not responses:
             return
 
+        if stream:
+            responses[stream]
+
         return RemoteServer.get_response(self.all_properties)
 
     def get_text(self):

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/webtoolkit/request.py

@@ -34,6 +34,7 @@ class PageRequestObject(object):
         respect_robots=None,
         accept_types=None,
         bytes_limit=None,
+        freshness_interval_s=None, # seconds
         http_proxy=None,
         https_proxy=None,
         settings=None,
@@ -54,6 +55,7 @@ class PageRequestObject(object):
         self.respect_robots = respect_robots
         self.accept_types = accept_types
         self.bytes_limit = bytes_limit
+        self.freshness_interval_s = freshness_interval_s
         self.http_proxy = http_proxy
         self.https_proxy = https_proxy
         self.settings = settings
@@ -125,6 +127,8 @@ class PageRequestObject(object):
             return False
         if self.bytes_limit != other.bytes_limit:
             return False
+        if self.freshness_interval_s != other.freshness_interval_s:
+            return False
         if self.http_proxy != other.http_proxy:
             return False
         if self.https_proxy != other.https_proxy:
@@ -176,6 +180,8 @@ def request_to_json(request):
         json["accept_types"] = request.accept_types
     if request.bytes_limit is not None:
         json["bytes_limit"] = request.bytes_limit
+    if request.freshness_interval_s is not None:
+        json["freshness_interval_s"] = request.freshness_interval_s
     if request.http_proxy is not None:
         json["http_proxy"] = request.http_proxy
     if request.https_proxy is not None:
@@ -233,6 +239,7 @@ def json_to_request(json_data):
     request.bytes_limit = json_data.get("bytes_limit")
     if request.bytes_limit is not None:
         request.bytes_limit = int(request.bytes_limit)
+    request.freshness_interval_s = json_data.get("freshness_interval_s")
     request.http_proxy = json_data.get("http_proxy")
     request.https_proxy = json_data.get("https_proxy")
     request.settings = json_data.get("settings")

{webtoolkit-0.1.67 → webtoolkit-0.1.70}/webtoolkit/statuses.py

@@ -45,8 +45,8 @@ HTTP_STATUS_CODE_TIMEOUT = 604                  # web page access timeout
 HTTP_STATUS_CODE_FILE_TOO_BIG = 612
 HTTP_STATUS_CODE_PAGE_UNSUPPORTED = 613
 HTTP_STATUS_CODE_SERVER_ERROR = 614
-HTTP_STATUS_CODE_SERVER_TOO_MANY_REQUESTS = 615 # this server too many requests
-HTTP_STATUS_CODE_SERVER_DATA_NOT_READY = 616
+HTTP_STATUS_CODE_SERVER_TOO_MANY_REQUESTS = 615 # this server too many requests, does not accept new
+HTTP_STATUS_CODE_SERVER_DATA_NOT_READY = 616    # we do not want to keep connections too long. Return not ready. Client can continue
 
 
 def status_code_to_text(status_code):