lacuscore 1.8.7__py3-none-any.whl → 1.8.9__py3-none-any.whl

lacuscore/lacuscore.py

@@ -10,6 +10,7 @@ import os
 import pickle
 import random
 import re
+import sys
 import time
 import unicodedata
 import zlib
@@ -28,6 +29,7 @@ from urllib.parse import urlsplit
 
 from dns import resolver
 from dns.exception import DNSException
+from dns.exception import Timeout as DNSTimeout
 
 from defang import refang  # type: ignore[import-untyped]
 from playwrightcapture import Capture, PlaywrightCaptureException
@@ -37,6 +39,11 @@ from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import DataError
 from ua_parser import user_agent_parser  # type: ignore[import-untyped]
 
+if sys.version_info < (3, 11):
+    from async_timeout import timeout
+else:
+    from asyncio import timeout
+
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -161,7 +168,7 @@ class LacusCore():
                  tor_proxy: str | None=None,
                  only_global_lookups: bool=True,
                  max_retries: int=3,
-                 loglevel: str='INFO') -> None:
+                 loglevel: str | int='INFO') -> None:
         self.master_logger = logging.getLogger(f'{self.__class__.__name__}')
         self.master_logger.setLevel(loglevel)
         self.redis = redis_connector
@@ -169,6 +176,10 @@ class LacusCore():
         self.tor_proxy = tor_proxy
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
+        self.dnsresolver: resolver.Resolver = resolver.Resolver()
+        self.dnsresolver.cache = resolver.Cache(30)
+        self.dnsresolver.timeout = 2
+        self.dnsresolver.lifetime = 3
 
         # NOTE: Remove in 1.8.* - clear old ongoing captures queue in case of need
         if self.redis.type('lacus:ongoing') in ['set', b'set']:  # type: ignore[no-untyped-call]
@@ -559,6 +570,10 @@ class LacusCore():
                             # not an IP, try resolving
                             try:
                                 ips_to_check = self.__get_ips(logger, splitted_url.hostname)
+                            except DNSTimeout as e:
+                                # for a timeout, we do not want to retry, as it is likely to timeout again
+                                result = {'error': f'DNS Timeout for "{splitted_url.hostname}": {e}'}
+                                raise CaptureError(f'DNS Timeout for "{splitted_url.hostname}": {e}')
                             except Exception as e:
                                 result = {'error': f'Issue with hostname resolution ({splitted_url.hostname}): {e}. Full URL: "{url}".'}
                                 raise CaptureError(f'Issue with hostname resolution ({splitted_url.hostname}): {e}. Full URL: "{url}".')
@@ -594,7 +609,8 @@ class LacusCore():
                         browser=browser_engine,
                         device_name=to_capture.get('device_name'),
                         proxy=proxy,
-                        general_timeout_in_sec=general_timeout) as capture:
+                        general_timeout_in_sec=general_timeout,
+                        loglevel=self.master_logger.getEffectiveLevel()) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.get('headers')  # type: ignore[assignment]
                     capture.cookies = to_capture.get('cookies')  # type: ignore[assignment]
@@ -606,18 +622,18 @@ class LacusCore():
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
                     try:
-                        await asyncio.wait_for(capture.initialize_context(), timeout=general_timeout)
+                        async with timeout(general_timeout):
+                            await capture.initialize_context()
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
                         logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
                         raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                    playwright_result = await asyncio.wait_for(
-                        capture.capture_page(
+                    async with timeout(self.max_capture_time):
+                        playwright_result = await capture.capture_page(
                             url, referer=to_capture.get('referer'),
                             depth=to_capture.get('depth', 0),
                             rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
                             with_favicon=to_capture.get('with_favicon', False),
-                            max_depth_capture_time=self.max_capture_time),
-                        timeout=self.max_capture_time)
+                            max_depth_capture_time=self.max_capture_time)
                     result = cast(CaptureResponse, playwright_result)
                     if 'error' in result and 'error_name' in result:
                         # generate stats
@@ -847,13 +863,17 @@ class LacusCore():
         # It is happening when the error code is NoAnswer
         resolved_ips = []
         try:
-            answers_a = resolver.resolve(hostname, 'A')
+            answers_a = self.dnsresolver.resolve(hostname, 'A')
             resolved_ips += [ip_address(str(answer)) for answer in answers_a]
+        except DNSTimeout as e:
+            raise e
         except DNSException as e:
             logger.info(f'No A record for "{hostname}": {e}')
         try:
-            answers_aaaa = resolver.resolve(hostname, 'AAAA')
+            answers_aaaa = self.dnsresolver.resolve(hostname, 'AAAA')
             resolved_ips += [ip_address(str(answer)) for answer in answers_aaaa]
+        except DNSTimeout as e:
+            raise e
         except DNSException as e:
             logger.info(f'No AAAA record for "{hostname}": {e}')
         return resolved_ips

{lacuscore-1.8.7.dist-info → lacuscore-1.8.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.8.7
+Version: 1.8.9
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -25,10 +25,11 @@ Classifier: Topic :: Security
 Provides-Extra: docs
 Requires-Dist: Sphinx (<7.2) ; (python_version < "3.9") and (extra == "docs")
 Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "docs")
+Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.23.8,<2.0.0)
-Requires-Dist: redis[hiredis] (>=5.0.1,<6.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.23.10,<2.0.0)
+Requires-Dist: redis[hiredis] (>=5.0.3,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/

lacuscore-1.8.9.dist-info/RECORD

@@ -0,0 +1,8 @@
+lacuscore/__init__.py,sha256=10JsUmjLD9xQIsjTDYT5nGowsJVe9C-FHNkaUFuAUtU,321
+lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
+lacuscore/lacuscore.py,sha256=iI1Cz9gnhQo8dw7gPII_vCsGHB_wzv-_9cpB3jRDizw,43901
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore-1.8.9.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.8.9.dist-info/METADATA,sha256=L1ITEAiio8MB6NUSn-Yi9tAL6BaYMnz-VVKFg_2IX-A,2629
+lacuscore-1.8.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lacuscore-1.8.9.dist-info/RECORD,,

{lacuscore-1.8.7.dist-info → lacuscore-1.8.9.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

lacuscore-1.8.7.dist-info/RECORD

@@ -1,8 +0,0 @@
-lacuscore/__init__.py,sha256=10JsUmjLD9xQIsjTDYT5nGowsJVe9C-FHNkaUFuAUtU,321
-lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
-lacuscore/lacuscore.py,sha256=R1IBAnasIRcCnx2Qykd7D3v8xr2nSddIvNuf2bcoZGc,42988
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore-1.8.7.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.8.7.dist-info/METADATA,sha256=hVV1r6OfAkZPwWnxsBd-Z329QfkxENhn9Ik22KnAdBM,2556
-lacuscore-1.8.7.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-lacuscore-1.8.7.dist-info/RECORD,,