lacuscore 1.10.9__tar.gz → 1.10.11__tar.gz

{lacuscore-1.10.9 → lacuscore-1.10.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.10.9
+Version: 1.10.11
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -30,7 +30,7 @@ Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
 Requires-Dist: eval-type-backport (>=0.2.0,<0.3.0) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.25.10,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.25.13,<2.0.0)
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.8,<6.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)

{lacuscore-1.10.9 → lacuscore-1.10.11}/lacuscore/lacuscore.py

@@ -23,7 +23,8 @@ from typing import Literal, Any, overload, cast, Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
-from dns import resolver
+from dns.resolver import Cache
+from dns.asyncresolver import Resolver
 from dns.exception import DNSException
 from dns.exception import Timeout as DNSTimeout
 
@@ -98,11 +99,14 @@ class LacusCore():
         self.tor_proxy = tor_proxy
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
-        self.dnsresolver: resolver.Resolver = resolver.Resolver()
-        self.dnsresolver.cache = resolver.Cache(30)
+        self.dnsresolver: Resolver = Resolver()
+        self.dnsresolver.cache = Cache(900)
         self.dnsresolver.timeout = 2
         self.dnsresolver.lifetime = 3
 
+        # Enable new chromium headless by default.
+        os.environ["PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW"] = "1"
+
         # NOTE: Remove in 1.8.* - clear old ongoing captures queue in case of need
         if self.redis.type('lacus:ongoing') in ['set', b'set']:  # type: ignore[no-untyped-call]
             self.redis.delete('lacus:ongoing')
@@ -411,7 +415,7 @@ class LacusCore():
                         except ValueError:
                             # not an IP, try resolving
                             try:
-                                ips_to_check = self.__get_ips(logger, splitted_url.hostname)
+                                ips_to_check = await self.__get_ips(logger, splitted_url.hostname)
                             except DNSTimeout as e:
                                 # for a timeout, we do not want to retry, as it is likely to timeout again
                                 result = {'error': f'DNS Timeout for "{splitted_url.hostname}": {e}'}
@@ -730,22 +734,40 @@ class LacusCore():
         p.zrem('lacus:ongoing', uuid)
         p.execute()
 
-    def __get_ips(self, logger: LacusCoreLogAdapter, hostname: str) -> list[IPv4Address | IPv6Address]:
+    async def __get_ips(self, logger: LacusCoreLogAdapter, hostname: str) -> list[IPv4Address | IPv6Address]:
         # We need to use dnspython for resolving because socket.getaddrinfo will sometimes be stuck for ~10s
         # It is happening when the error code is NoAnswer
         resolved_ips = []
-        try:
-            answers_a = self.dnsresolver.resolve(hostname, 'A')
-            resolved_ips += [ip_address(str(answer)) for answer in answers_a]
-        except DNSTimeout as e:
-            raise e
-        except DNSException as e:
-            logger.info(f'No A record for "{hostname}": {e}')
-        try:
-            answers_aaaa = self.dnsresolver.resolve(hostname, 'AAAA')
-            resolved_ips += [ip_address(str(answer)) for answer in answers_aaaa]
-        except DNSTimeout as e:
-            raise e
-        except DNSException as e:
-            logger.info(f'No AAAA record for "{hostname}": {e}')
+        max_timeout_retries = 3
+        _current_retries = 0
+        while _current_retries < max_timeout_retries:
+            _current_retries += 1
+            try:
+                answers_a = await self.dnsresolver.resolve(hostname, 'A')
+                resolved_ips += [ip_address(str(answer)) for answer in answers_a]
+            except DNSTimeout as e:
+                if _current_retries < max_timeout_retries:
+                    logger.info(f'DNS Timeout for "{hostname}" (A record), retrying.')
+                    await asyncio.sleep(1)
+                    continue
+                raise e
+            except DNSException as e:
+                logger.info(f'No A record for "{hostname}": {e}')
+            break
+
+        _current_retries = 0
+        while _current_retries < max_timeout_retries:
+            _current_retries += 1
+            try:
+                answers_aaaa = await self.dnsresolver.resolve(hostname, 'AAAA')
+                resolved_ips += [ip_address(str(answer)) for answer in answers_aaaa]
+            except DNSTimeout as e:
+                if _current_retries < max_timeout_retries:
+                    logger.info(f'DNS Timeout for "{hostname}" (AAAA record), retrying.')
+                    await asyncio.sleep(1)
+                    continue
+                raise e
+            except DNSException as e:
+                logger.info(f'No AAAA record for "{hostname}": {e}')
+            break
         return resolved_ips

{lacuscore-1.10.9 → lacuscore-1.10.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.10.9"
+version = "1.10.11"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -35,7 +35,7 @@ Sphinx = [
    {version = "^7.2", python = ">=3.9,<3.10", optional = true},
    {version = "^8", python = ">=3.10", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.25.10"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.25.13"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
 redis = {version = "^5.0.8", extras = ["hiredis"]}
@@ -48,7 +48,7 @@ eval-type-backport = {version = "^0.2.0", python = "<3.10"}
 docs = ["Sphinx"]
 
 [tool.poetry.group.dev.dependencies]
-types-redis = {version = "^4.6.0.20240726"}
+types-redis = {version = "^4.6.0.20240806"}
 mypy = "^1.11.1"
 types-requests = "^2.32.0.20240712"
 types-beautifulsoup4 = "^4.12.0.20240511"