webtoolkit 0.1.57__tar.gz → 0.1.58__tar.gz

{webtoolkit-0.1.57 → webtoolkit-0.1.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.1.57
+Version: 0.1.58
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny

{webtoolkit-0.1.57 → webtoolkit-0.1.58}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.1.57"
+version = "0.1.58"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.1.57 → webtoolkit-0.1.58}/webtoolkit/baseurl.py

@@ -550,7 +550,7 @@ class BaseUrl(ContentInterface):
         request_data["crawler_type"] = type(request_data["crawler_type"]).__name__
         all_properties.append({"name": "Request", "data": request_data})
 
-        response_data = self.get_response_data(streams)
+        response_data = self.get_response_data()
         all_properties.append({"name": "Response", "data": response_data})
         if response:
             raw_headers_data = response.get_headers()
@@ -639,10 +639,14 @@ class BaseUrl(ContentInterface):
 
         return response_data
 
-    def get_response_data(self, streams):
+    def get_response_data(self):
         """
         Returns response data
         Easy digestible response data
+
+        It is handler responsibility to provide response and streams.
+        Main response should not be valid if any stream is not valid.
+        Hash should also be properly calculated
         """
         response = self.get_response()
         response_data = self.response_to_data(response)

{webtoolkit-0.1.57 → webtoolkit-0.1.58}/webtoolkit/handlers/defaulturlhandler.py

@@ -145,9 +145,8 @@ class DefaultCompoundChannelHandler(DefaultChannelHandler):
     def get_response(self):
         """
         There can be many responses.
-        If at least one is invalid - return it
-
-        If responses are valid return best match
+         - If at least one is invalid - return it. Maybe we could merge responses somehow
+         - If responses are valid return best match
         """
         valid_responses = []
         invalid_responses = []

{webtoolkit-0.1.57 → webtoolkit-0.1.58}/webtoolkit/remoteserver.py

@@ -235,6 +235,10 @@ class RemoteServer(object):
         return responses
 
     def get_response(all_properties):
+        """
+        Returns response from all properties
+        """
+
         """
         responses = RemoteServer.get_responses(all_properties)
         if not responses:
@@ -256,6 +260,7 @@ class RemoteServer(object):
         for url in responses:
             return responses[url]
         """
+
         response_data = RemoteServer.read_properties_section("Response", all_properties)
         response = json_to_response(response_data)
         return response