lacuscore 1.9.3__tar.gz → 1.9.5__tar.gz

{lacuscore-1.9.3 → lacuscore-1.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.3
+Version: 1.9.5
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,9 +28,9 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.6,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.11,<2.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/ail-project/LacusCore

{lacuscore-1.9.3 → lacuscore-1.9.5}/lacuscore/lacuscore.py

@@ -43,9 +43,19 @@ from .helpers import (
 
 if sys.version_info < (3, 11):
     from async_timeout import timeout
+
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired:
+            logger.warning(f'Timeout expired: {error_message}')
+
 else:
     from asyncio import timeout
 
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired():
+            logger.warning(f'Timeout expired: {error_message}')
+
+
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -523,15 +533,12 @@ class LacusCore():
                     browser_engine = 'webkit'
             try:
                 logger.debug(f'Capturing {url}')
-                # NOTE: starting with python 3.11, we can use asyncio.timeout
-                # async with asyncio.timeout(self.max_capture_time):
-                general_timeout = to_capture.get('general_timeout_in_sec')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
                         browser=browser_engine,
                         device_name=to_capture.get('device_name'),
                         proxy=proxy,
-                        general_timeout_in_sec=general_timeout,
+                        general_timeout_in_sec=to_capture.get('general_timeout_in_sec'),
                         loglevel=self.master_logger.getEffectiveLevel(),
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
@@ -544,20 +551,30 @@ class LacusCore():
                     capture.timezone_id = to_capture.get('timezone_id')  # type: ignore[assignment]
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
+
+                    # make sure the initialization doesn't take too long
+                    init_timeout = max(self.max_capture_time / 10, 5)
                     try:
-                        async with timeout(general_timeout):
+                        async with timeout(init_timeout) as initialize_timeout:
                             await capture.initialize_context()
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
-                        logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                        raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                    async with timeout(self.max_capture_time):
-                        playwright_result = await capture.capture_page(
-                            url, referer=to_capture.get('referer'),
-                            depth=to_capture.get('depth', 0),
-                            rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
-                            with_favicon=to_capture.get('with_favicon', False),
-                            allow_tracking=to_capture.get('allow_tracking', False),
-                            max_depth_capture_time=self.max_capture_time)
+                        timeout_expired(initialize_timeout, logger, 'Initializing took too long.')
+                        logger.warning(f'Initializing the context for {url} took longer than the allowed initialization timeout ({init_timeout}s)')
+                        raise RetryCapture(f'Initializing the context for {url} took longer than the allowed initialization timeout ({init_timeout}s)')
+
+                    try:
+                        async with timeout(self.max_capture_time) as capture_timeout:
+                            playwright_result = await capture.capture_page(
+                                url, referer=to_capture.get('referer'),
+                                depth=to_capture.get('depth', 0),
+                                rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
+                                with_favicon=to_capture.get('with_favicon', False),
+                                allow_tracking=to_capture.get('allow_tracking', False),
+                                max_depth_capture_time=self.max_capture_time)
+                    except (TimeoutError, asyncio.exceptions.TimeoutError):
+                        timeout_expired(capture_timeout, logger, 'Capture took too long.')
+                        logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
+                        raise RetryCapture(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
                     result = cast(CaptureResponse, playwright_result)
                     if 'error' in result and 'error_name' in result:
                         # generate stats
@@ -575,10 +592,6 @@ class LacusCore():
                 # The capture can be canceled if it has been running for way too long.
                 # We can give it another short.
                 raise RetryCapture(f'The capture of {url} has been cancelled.')
-            except (TimeoutError, asyncio.exceptions.TimeoutError):
-                logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
-                result = {'error': f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)'}
-                raise CaptureError(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
             except Exception as e:
                 logger.exception(f'Something went poorly {url} - {e}')
                 result = {'error': f'Something went poorly {url} - {e}'}

{lacuscore-1.9.3 → lacuscore-1.9.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.9.3"
+version = "1.9.5"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -29,12 +29,12 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
-requests = "^2.31.0"
+requests = "^2.32.3"
 Sphinx = [
    {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.24.6"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.24.11"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
 redis = {version = "^5.0.4", extras = ["hiredis"]}
@@ -47,14 +47,14 @@ docs = ["Sphinx"]
 [tool.poetry.group.dev.dependencies]
 types-redis = {version = "^4.6.0.20240425"}
 mypy = "^1.10.0"
-types-requests = "^2.31.0.20240406"
-types-beautifulsoup4 = "^4.12.0.20240229"
+types-requests = "^2.32.0.20240523"
+types-beautifulsoup4 = "^4.12.0.20240511"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-pytest = "^8.2.0"
+pytest = "^8.2.1"
 
 [build-system]
 requires = ["poetry_core"]