webtoolkit 0.1.48__tar.gz → 0.1.50__tar.gz

{webtoolkit-0.1.48 → webtoolkit-0.1.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.1.48
+Version: 0.1.50
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny

{webtoolkit-0.1.48 → webtoolkit-0.1.50}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.1.48"
+version = "0.1.50"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.1.48 → webtoolkit-0.1.50}/webtoolkit/baseurl.py

@@ -32,7 +32,7 @@ from .urllocation import (
 )
 
 from .statuses import status_code_to_text
-from .response import response_to_json
+from .response import response_to_json, PageResponseObject
 from .request import request_to_json, PageRequestObject
 from .handlers import (
     HandlerInterface,
@@ -234,6 +234,11 @@ class BaseUrl(ContentInterface):
 
             return self.response
 
+        location = UrlLocation(self.request.url)
+        if location.is_onion():
+            self.response = PageResponseObject(url=self.request.url, status_code=0)
+            return self.response
+
     def get_streams(self):
         """
         Returns all responses
@@ -275,6 +280,12 @@ class BaseUrl(ContentInterface):
             return
 
         p = UrlLocation(url)
+        if p.is_onion():
+            """
+            Currently there is no handler to support onions
+            """
+            return
+
         short_url = p.get_protocolless()
 
         if not short_url:

{webtoolkit-0.1.48 → webtoolkit-0.1.50}/webtoolkit/tests/fakeinternet.py

@@ -8,6 +8,7 @@ This module provides replacement for the Internet.
 import logging
 import unittest
 import traceback
+import os
 
 from webtoolkit.utils.dateutils import DateUtils
 from webtoolkit import (
@@ -143,6 +144,10 @@ class FakeInternetTestCase(unittest.TestCase):
         infos = AppLogging.objects.filter(level=int(logging.ERROR))
         return infos.count() == 0
 
+    def is_memory_test(self):
+        test_memory = os.getenv("TEST_MEMORY")
+        return test_memory
+
     def create_example_data(self):
         self.create_example_sources()
         self.create_example_links()

{webtoolkit-0.1.48 → webtoolkit-0.1.50}/webtoolkit/urllocation.py

@@ -3,6 +3,7 @@ Internet location parsing and processing.
 """
 
 from urllib.parse import unquote, urlparse, parse_qs
+import base64
 import mimetypes
 import ipaddress
 
@@ -538,6 +539,7 @@ class UrlLocation(object):
         url = UrlLocation.get_google_redirect_fix2(url)
         url = UrlLocation.get_youtube_redirect_fix(url)
         url = UrlLocation.get_linkedin_redirect_fix(url)
+        url = UrlLocation.get_bing_redirect_fix(url)
         url = UrlLocation.get_trackless_url(url)
 
         return url.url
@@ -613,6 +615,30 @@ class UrlLocation(object):
 
         return url
 
+    def get_bing_redirect_fix(bing_url):
+        try:
+            parsed = urlparse(bing_url)
+            params = parse_qs(parsed.query)
+
+            encoded = params.get("u", [None])[0]
+            if not encoded:
+                return bing_url
+
+            # Remove prefix like 'a1'
+            if encoded.startswith("a1"):
+                encoded = encoded[2:]
+
+            # handle missing padding
+            padding = '=' * (-len(encoded) % 4)
+            decoded_bytes = base64.b64decode(encoded + padding)
+
+            url = decoded_bytes.decode("utf-8", errors="ignore")
+            url = UrlLocation.get_cleaned_link(url)
+            return url
+
+        except Exception:
+            return bing_url
+
     def get_url_arg(self):
         url = self.url
         if not url: