webtoolkit 0.1.48__tar.gz → 0.1.49__tar.gz

{webtoolkit-0.1.48 → webtoolkit-0.1.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: webtoolkit
-Version: 0.1.48
+Version: 0.1.49
 Summary: Web tools and interfaces for Internet data processing.
 License: GPL3
 Author: Iwan Grozny

{webtoolkit-0.1.48 → webtoolkit-0.1.49}/pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "webtoolkit"
-version = "0.1.48"
+version = "0.1.49"
 description = "Web tools and interfaces for Internet data processing."
 authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
 license = "GPL3"

{webtoolkit-0.1.48 → webtoolkit-0.1.49}/webtoolkit/tests/fakeinternet.py

@@ -8,6 +8,7 @@ This module provides replacement for the Internet.
 import logging
 import unittest
 import traceback
+import os
 
 from webtoolkit.utils.dateutils import DateUtils
 from webtoolkit import (
@@ -143,6 +144,10 @@ class FakeInternetTestCase(unittest.TestCase):
         infos = AppLogging.objects.filter(level=int(logging.ERROR))
         return infos.count() == 0
 
+    def is_memory_test(self):
+        test_memory = os.getenv("TEST_MEMORY")
+        return test_memory
+
     def create_example_data(self):
         self.create_example_sources()
         self.create_example_links()

{webtoolkit-0.1.48 → webtoolkit-0.1.49}/webtoolkit/urllocation.py

@@ -3,6 +3,7 @@ Internet location parsing and processing.
 """
 
 from urllib.parse import unquote, urlparse, parse_qs
+import base64
 import mimetypes
 import ipaddress
 
@@ -538,6 +539,7 @@ class UrlLocation(object):
         url = UrlLocation.get_google_redirect_fix2(url)
         url = UrlLocation.get_youtube_redirect_fix(url)
         url = UrlLocation.get_linkedin_redirect_fix(url)
+        url = UrlLocation.get_bing_redirect_fix(url)
         url = UrlLocation.get_trackless_url(url)
 
         return url.url
@@ -613,6 +615,30 @@ class UrlLocation(object):
 
         return url
 
+    def get_bing_redirect_fix(bing_url):
+        try:
+            parsed = urlparse(bing_url)
+            params = parse_qs(parsed.query)
+
+            encoded = params.get("u", [None])[0]
+            if not encoded:
+                return bing_url
+
+            # Remove prefix like 'a1'
+            if encoded.startswith("a1"):
+                encoded = encoded[2:]
+
+            # handle missing padding
+            padding = '=' * (-len(encoded) % 4)
+            decoded_bytes = base64.b64decode(encoded + padding)
+
+            url = decoded_bytes.decode("utf-8", errors="ignore")
+            url = UrlLocation.get_cleaned_link(url)
+            return url
+
+        except Exception:
+            return bing_url
+
     def get_url_arg(self):
         url = self.url
         if not url: