PlaywrightCapture 1.28.6__tar.gz → 1.29.0__tar.gz

{playwrightcapture-1.28.6 → playwrightcapture-1.29.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: PlaywrightCapture
-Version: 1.28.6
+Version: 1.29.0
 Summary: A simple library to capture websites using playwright
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -20,15 +20,17 @@ Classifier: Topic :: Security
 Provides-Extra: recaptcha
 Requires-Dist: SpeechRecognition (>=3.14.2) ; extra == "recaptcha"
 Requires-Dist: aiohttp-socks (>=0.10.1)
-Requires-Dist: aiohttp[speedups] (>=3.11.16)
+Requires-Dist: aiohttp[speedups] (>=3.11.18)
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
-Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.3)
+Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
 Requires-Dist: dateparser (>=1.2.1)
+Requires-Dist: dnspython (>=2.7.0,<3.0.0)
 Requires-Dist: playwright (>=1.51.0)
 Requires-Dist: playwright-stealth (>=1.0.6)
 Requires-Dist: puremagic (>=1.28)
 Requires-Dist: pydub (>=0.25.1) ; extra == "recaptcha"
-Requires-Dist: setuptools (>=78.1.0)
+Requires-Dist: python-socks (>=2.7.1,<3.0.0)
+Requires-Dist: setuptools (>=79.0.0)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0) ; python_version < "3.12"
 Requires-Dist: tzdata (>=2025.2)
 Requires-Dist: w3lib (>=2.3.1)

{playwrightcapture-1.28.6 → playwrightcapture-1.29.0}/playwrightcapture/capture.py

@@ -37,6 +37,7 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from .exceptions import UnknownPlaywrightBrowser, UnknownPlaywrightDevice, InvalidPlaywrightParameter
+from .socks5dnslookup import Socks5Resolver
 
 from zoneinfo import available_timezones
 all_timezones_set = available_timezones()
@@ -144,6 +145,7 @@ class Capture():
 
     def __init__(self, browser: BROWSER | None=None, device_name: str | None=None,
                  proxy: str | dict[str, str] | None=None,
+                 socks5_dns_resolver: str | list[str] | None=None,
                  general_timeout_in_sec: int | None=None, loglevel: str | int='INFO',
                  uuid: str | None=None, headless: bool=True):
         """Captures a page with Playwright.
@@ -151,6 +153,7 @@ class Capture():
         :param browser: The browser to use for the capture.
         :param device_name: The pre-defined device to use for the capture (from playwright).)
         :param proxy: The external proxy to use for the capture.
+        :param socks5_dns_resolver: DNS resolver to use for the socks5 proxy and fill the HAR file.
         :param general_timeout_in_sec: The general timeout for the capture, including children.
         :param loglevel: Python loglevel
         :param uuid: The UUID of the capture.
@@ -177,6 +180,7 @@ class Capture():
         self.device_name: str | None = device_name
         self.headless: bool = headless
         self.proxy: ProxySettings = {}
+        self.socks5_dns_resolver: str | list[str] | None = socks5_dns_resolver
         if proxy:
             if isinstance(proxy, str):
                 self.proxy = self.__prepare_proxy_playwright(proxy)
@@ -1238,6 +1242,12 @@ class Capture():
                         with open(self._temp_harfile.name) as _har:
                             to_return['har'] = json.load(_har)
                         self.logger.debug('Got HAR.')
+                    if (to_return.get('har') and self.proxy and self.proxy.get('server')
+                            and self.proxy['server'].startswith('socks5')):
+                        # Only if the capture was not done via a socks5 proxy
+                        if har := to_return['har']:  # Could be None
+                            async with timeout(30):
+                                await self.socks5_resolver(har)
                 except (TimeoutError, asyncio.TimeoutError):
                     self.logger.warning("Unable to close page and context at the end of the capture.")
                     errors.append("Unable to close page and context at the end of the capture.")
@@ -1713,3 +1723,31 @@ class Capture():
         return to_return
 
     # END FAVICON EXTRACTOR
+
+    # ##### Run DNS resolution over socks5 proxy #####
+    # This is only use when the capture is done over a socks5 proxy, and not on a .onion
+    # We get the HAR file, iterate over the entries an update the IPs
+
+    async def socks5_resolver(self, harfile: dict[str, Any]) -> None:
+        resolver = Socks5Resolver(logger=self.logger, socks5_proxy=self.proxy['server'],
+                                  dns_resolver=self.socks5_dns_resolver)
+        # get all the hostnames from the HAR file
+        hostnames = set()
+        for entry in harfile['log']['entries']:
+            if entry['request']['url']:
+                parsed = urlparse(entry['request']['url'])
+                if parsed.netloc and not parsed.netloc.endswith('onion'):
+                    hostnames.add(parsed.netloc)
+        # use the same technique as in lookyloo to resolve many domains in parallel
+        semaphore = asyncio.Semaphore(20)
+        all_requests = [resolver.resolve(hostname, semaphore) for hostname in hostnames]
+        await asyncio.gather(*all_requests)
+        self.logger.debug('Resolved all domains through the proxy.')
+        for entry in harfile['log']['entries']:
+            if entry['request']['url']:
+                parsed = urlparse(entry['request']['url'])
+                if parsed.netloc and not parsed.netloc.endswith('onion'):
+                    answer = resolver.get_cache(parsed.netloc)
+                    if answer:
+                        entry['serverIPAddress'] = {str(b) for b in answer}.pop()
+        self.logger.debug('Done updating HAR file')

playwrightcapture-1.29.0/playwrightcapture/socks5dnslookup.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import asyncio
+import socket
+
+from typing import TYPE_CHECKING
+
+import dns
+
+from dns.asyncresolver import Resolver
+from dns.resolver import Cache
+from dns._asyncio_backend import _maybe_wait_for, StreamSocket, Backend
+from python_socks.async_.asyncio import Proxy
+
+
+if TYPE_CHECKING:
+    from .capture import PlaywrightCaptureLogAdapter
+    from logging import Logger
+
+
+class Socks5Backend(Backend):
+
+    def __init__(self, socks5_proxy_url: str):
+        super().__init__()
+        self.proxy = Proxy.from_url(socks5_proxy_url)
+
+    def name(self) -> str:
+        return "asyncio socks5"
+
+    async def make_socket(  # type: ignore[no-untyped-def]
+        self,
+        af,
+        socktype,
+        proto=0,
+        source=None,
+        destination=None,
+        timeout=None,
+        ssl_context=None,
+        server_hostname=None,
+    ):
+        if socktype == socket.SOCK_STREAM:
+            if destination is None:
+                # This shouldn't happen, but we check to make code analysis software
+                # happier.
+                raise ValueError("destination required for stream sockets")
+            sock = await self.proxy.connect(dest_host=destination[0], dest_port=destination[1])
+            (r, w) = await _maybe_wait_for(  # type: ignore[no-untyped-call]
+                asyncio.open_connection(
+                    None,
+                    None,
+                    sock=sock,
+                    ssl=ssl_context,
+                    family=af,
+                    proto=proto,
+                    local_addr=source,
+                    server_hostname=server_hostname,
+                ),
+                timeout,
+            )
+            return StreamSocket(af, r, w)  # type: ignore[no-untyped-call]
+        raise NotImplementedError(
+            "unsupported socket " + f"type {socktype}"
+        )  # pragma: no cover
+
+
+class Socks5Resolver:
+
+    def __init__(self, logger: Logger | PlaywrightCaptureLogAdapter, socks5_proxy: str, dns_resolver: str | list[str] | None=None):
+        self.logger = logger
+        # configure set to false means we don't want to load resolv.conf
+        self.resolver = Resolver(configure=False)
+        self.resolver.cache = Cache(900)
+        self.resolver.timeout = 2
+        self.resolver.lifetime = 4
+
+        if not dns_resolver:
+            # Fallback to 1.1.1.1
+            dns_resolver = ['1.1.1.1']
+        elif isinstance(dns_resolver, str):
+            dns_resolver = [dns_resolver]
+        self.resolver.nameservers = dns_resolver
+
+        self.backend = Socks5Backend(socks5_proxy_url=socks5_proxy)
+
+    def get_cache(self, domain: str, rdatatype: dns.rdatatype.RdataType=dns.rdatatype.A) -> dns.resolver.Answer | None:
+        # Get domain from cache
+        return self.resolver.cache.get((dns.name.from_text(domain), rdatatype, dns.rdataclass.IN))
+
+    async def resolve(self, domain: str, semaphore: asyncio.Semaphore, rdatatype: dns.rdatatype.RdataType=dns.rdatatype.A) -> dns.resolver.Answer | None:
+        # Resolve the A record only for the domain, might want to do AAAA instead.
+        async with semaphore:
+            max_retries = 3
+            while max_retries > 0:
+                try:
+                    response = await self.resolver.resolve(domain, rdatatype,
+                                                           tcp=True, backend=self.backend)
+                    return response
+                except dns.resolver.LifetimeTimeout:
+                    # Retry a few times on timeout, it happens.
+                    max_retries -= 1
+                    if max_retries > 0:
+                        self.logger.debug(f"[Socks5] Timeout resolving {domain}, retrying.")
+                        await asyncio.sleep(1)
+                    else:
+                        self.logger.info(f"[Socks5] Timeout resolving {domain}.")
+                except Exception as e:
+                    self.logger.info(f"[Socks5] Error resolving {domain}: {e}")
+                    break
+            return None

{playwrightcapture-1.28.6 → playwrightcapture-1.29.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "PlaywrightCapture"
-version = "1.28.6"
+version = "1.29.0"
 description = "A simple library to capture websites using playwright"
 authors = [
     {name="Raphaël Vinot", email= "raphael.vinot@circl.lu"}
@@ -14,16 +14,18 @@ dynamic = [ "classifiers" ]
 dependencies = [
     "playwright (>=1.51.0)",
     "dateparser (>=1.2.1)",
-    "beautifulsoup4[charset-normalizer,lxml] (>=4.13.3)",
+    "beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)",
     "w3lib (>=2.3.1)",
     "tzdata (>=2025.2)",
     "playwright-stealth (>=1.0.6)",
-    "setuptools (>=78.1.0)",
+    "setuptools (>=79.0.0)",
     "puremagic (>=1.28)",
     "async-timeout (>=5.0.1) ; python_version < \"3.11\"",
-    "aiohttp[speedups] (>=3.11.16)",
+    "aiohttp[speedups] (>=3.11.18)",
     "aiohttp-socks (>=0.10.1)",
-    "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.12\""
+    "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.12\"",
+    "dnspython (>=2.7.0,<3.0.0)",
+    "python-socks (>=2.7.1,<3.0.0)"
 ]
 
 [project.urls]