scrapling 0.2.99__py3-none-any.whl → 0.3.1__py3-none-any.whl

scrapling/engines/toolbelt/navigation.py

@@ -1,74 +1,97 @@
 """
 Functions related to files and URLs
 """
-import os
+
+from pathlib import Path
+from functools import lru_cache
 from urllib.parse import urlencode, urlparse
 
 from playwright.async_api import Route as async_Route
+from msgspec import Struct, structs, convert, ValidationError
 from playwright.sync_api import Route
 
-from scrapling.core._types import Dict, Optional, Union
-from scrapling.core.utils import log, lru_cache
+from scrapling.core.utils import log
+from scrapling.core._types import Dict, Optional, Tuple
 from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
 
+__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
+
+
+class ProxyDict(Struct):
+    server: str
+    username: str = ""
+    password: str = ""
+
 
 def intercept_route(route: Route):
-    """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+    """This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
 
     :param route: PlayWright `Route` object of the current page
     :return: PlayWright `Route` object
     """
     if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
-        log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        log.debug(
+            f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
+        )
         route.abort()
     else:
         route.continue_()
 
 
 async def async_intercept_route(route: async_Route):
-    """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+    """This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
 
     :param route: PlayWright `Route` object of the current page
     :return: PlayWright `Route` object
     """
     if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
-        log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+        log.debug(
+            f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
+        )
         await route.abort()
     else:
         await route.continue_()
 
 
-def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
+def construct_proxy_dict(
+    proxy_string: str | Dict[str, str], as_tuple=False
+) -> Optional[Dict | Tuple]:
     """Validate a proxy and return it in the acceptable format for Playwright
     Reference: https://playwright.dev/python/docs/network#http-proxy
 
     :param proxy_string: A string or a dictionary representation of the proxy.
+    :param as_tuple: Return the proxy dictionary as a tuple to be cachable
     :return:
     """
-    if proxy_string:
-        if isinstance(proxy_string, str):
-            proxy = urlparse(proxy_string)
-            try:
-                return {
-                    'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
-                    'username': proxy.username or '',
-                    'password': proxy.password or '',
-                }
-            except ValueError:
-                # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
-                raise TypeError('The proxy argument\'s string is in invalid format!')
-
-        elif isinstance(proxy_string, dict):
-            valid_keys = ('server', 'username', 'password', )
-            if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
-                return proxy_string
-            else:
-                raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
-
-        else:
-            raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
-
-    # The default value for proxy in Playwright's source is `None`
+    if isinstance(proxy_string, str):
+        proxy = urlparse(proxy_string)
+        if (
+            proxy.scheme not in ("http", "https", "socks4", "socks5")
+            or not proxy.hostname
+        ):
+            raise ValueError("Invalid proxy string!")
+
+        try:
+            result = {
+                "server": f"{proxy.scheme}://{proxy.hostname}",
+                "username": proxy.username or "",
+                "password": proxy.password or "",
+            }
+            if proxy.port:
+                result["server"] += f":{proxy.port}"
+            return tuple(result.items()) if as_tuple else result
+        except ValueError:
+            # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
+            raise ValueError("The proxy argument's string is in invalid format!")
+
+    elif isinstance(proxy_string, dict):
+        try:
+            validated = convert(proxy_string, ProxyDict)
+            result_dict = structs.asdict(validated)
+            return tuple(result_dict.items()) if as_tuple else result_dict
+        except ValidationError as e:
+            raise TypeError(f"Invalid proxy dictionary: {e}")
+
     return None
 
 
@@ -84,17 +107,24 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         parsed = urlparse(cdp_url)
 
         # Check scheme
-        if parsed.scheme not in ('ws', 'wss'):
+        if parsed.scheme not in ("ws", "wss"):
             raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
 
         # Validate hostname and port
         if not parsed.netloc:
             raise ValueError("Invalid hostname for the CDP URL")
 
-        # Ensure path starts with /
+        try:
+            # Checking if the port is valid (if available)
+            _ = parsed.port
+        except ValueError:
+            # urlparse will raise `ValueError` if the port can't be casted to integer
+            raise ValueError("Invalid port for the CDP URL")
+
+        # Ensure the path starts with /
         path = parsed.path
-        if not path.startswith('/'):
-            path = '/' + path
+        if not path.startswith("/"):
+            path = "/" + path
 
         # Reconstruct the base URL with validated parts
         validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
@@ -112,10 +142,9 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
 
 @lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
-    """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
+    """Takes the base filename of a JS file inside the `bypasses` folder, then return the full path of it
 
     :param filename: The base filename of the JS file.
     :return: The full path of the JS file.
     """
-    current_directory = os.path.dirname(__file__)
-    return os.path.join(current_directory, 'bypasses', filename)
+    return str(__BYPASSES_DIR__ / filename)