lacuscore 1.8.8__tar.gz → 1.8.9__tar.gz

{lacuscore-1.8.8 → lacuscore-1.8.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.8.8
+Version: 1.8.9
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,8 +28,8 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.23.9,<2.0.0)
-Requires-Dist: redis[hiredis] (>=5.0.2,<6.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.23.10,<2.0.0)
+Requires-Dist: redis[hiredis] (>=5.0.3,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/

{lacuscore-1.8.8 → lacuscore-1.8.9}/lacuscore/lacuscore.py

@@ -29,6 +29,7 @@ from urllib.parse import urlsplit
 
 from dns import resolver
 from dns.exception import DNSException
+from dns.exception import Timeout as DNSTimeout
 
 from defang import refang  # type: ignore[import-untyped]
 from playwrightcapture import Capture, PlaywrightCaptureException
@@ -175,6 +176,10 @@ class LacusCore():
         self.tor_proxy = tor_proxy
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
+        self.dnsresolver: resolver.Resolver = resolver.Resolver()
+        self.dnsresolver.cache = resolver.Cache(30)
+        self.dnsresolver.timeout = 2
+        self.dnsresolver.lifetime = 3
 
         # NOTE: Remove in 1.8.* - clear old ongoing captures queue in case of need
         if self.redis.type('lacus:ongoing') in ['set', b'set']:  # type: ignore[no-untyped-call]
@@ -565,6 +570,10 @@ class LacusCore():
                             # not an IP, try resolving
                             try:
                                 ips_to_check = self.__get_ips(logger, splitted_url.hostname)
+                            except DNSTimeout as e:
+                                # for a timeout, we do not want to retry, as it is likely to timeout again
+                                result = {'error': f'DNS Timeout for "{splitted_url.hostname}": {e}'}
+                                raise CaptureError(f'DNS Timeout for "{splitted_url.hostname}": {e}')
                             except Exception as e:
                                 result = {'error': f'Issue with hostname resolution ({splitted_url.hostname}): {e}. Full URL: "{url}".'}
                                 raise CaptureError(f'Issue with hostname resolution ({splitted_url.hostname}): {e}. Full URL: "{url}".')
@@ -854,13 +863,17 @@ class LacusCore():
         # It is happening when the error code is NoAnswer
         resolved_ips = []
         try:
-            answers_a = resolver.resolve(hostname, 'A')
+            answers_a = self.dnsresolver.resolve(hostname, 'A')
             resolved_ips += [ip_address(str(answer)) for answer in answers_a]
+        except DNSTimeout as e:
+            raise e
         except DNSException as e:
             logger.info(f'No A record for "{hostname}": {e}')
         try:
-            answers_aaaa = resolver.resolve(hostname, 'AAAA')
+            answers_aaaa = self.dnsresolver.resolve(hostname, 'AAAA')
             resolved_ips += [ip_address(str(answer)) for answer in answers_aaaa]
+        except DNSTimeout as e:
+            raise e
         except DNSException as e:
             logger.info(f'No AAAA record for "{hostname}": {e}')
         return resolved_ips

{lacuscore-1.8.8 → lacuscore-1.8.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.8.8"
+version = "1.8.9"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -34,10 +34,10 @@ Sphinx = [
    {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.23.9"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.23.10"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
-redis = {version = "^5.0.2", extras = ["hiredis"]}
+redis = {version = "^5.0.3", extras = ["hiredis"]}
 dnspython = "^2.6.1"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 
@@ -45,16 +45,16 @@ async-timeout = {version = "^4.0.3", python = "<3.11"}
 docs = ["Sphinx"]
 
 [tool.poetry.group.dev.dependencies]
-types-redis = {version = "^4.6.0.20240218"}
-mypy = "^1.8.0"
-types-requests = "^2.31.0.20240218"
+types-redis = {version = "^4.6.0.20240311"}
+mypy = "^1.9.0"
+types-requests = "^2.31.0.20240311"
 types-beautifulsoup4 = "^4.12.0.20240229"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-pytest = "^8.0.2"
+pytest = "^8.1.1"
 
 [build-system]
 requires = ["poetry_core"]