lacuscore 1.9.3__tar.gz → 1.9.4__tar.gz

{lacuscore-1.9.3 → lacuscore-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.3
+Version: 1.9.4
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,9 +28,9 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.6,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.10,<2.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: requests (>=2.32.1,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/ail-project/LacusCore

{lacuscore-1.9.3 → lacuscore-1.9.4}/lacuscore/lacuscore.py

@@ -43,9 +43,19 @@ from .helpers import (
 
 if sys.version_info < (3, 11):
     from async_timeout import timeout
+
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired:
+            logger.warning(f'Timeout expired: {error_message}')
+
 else:
     from asyncio import timeout
 
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired():
+            logger.warning(f'Timeout expired: {error_message}')
+
+
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -523,8 +533,6 @@ class LacusCore():
                     browser_engine = 'webkit'
             try:
                 logger.debug(f'Capturing {url}')
-                # NOTE: starting with python 3.11, we can use asyncio.timeout
-                # async with asyncio.timeout(self.max_capture_time):
                 general_timeout = to_capture.get('general_timeout_in_sec')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
@@ -545,12 +553,19 @@ class LacusCore():
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
                     try:
-                        async with timeout(general_timeout):
+                        # make sure the initialization doesn't take too long
+                        if general_timeout is None:
+                            general_timeout = 5
+                        init_timeout = max(general_timeout / 2, 5)
+                        async with timeout(init_timeout) as initialize_timeout:
                             await capture.initialize_context()
+
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
+                        timeout_expired(initialize_timeout, logger, 'Initializing took too long.')
                         logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
                         raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                    async with timeout(self.max_capture_time):
+
+                    async with timeout(self.max_capture_time) as capture_timeout:
                         playwright_result = await capture.capture_page(
                             url, referer=to_capture.get('referer'),
                             depth=to_capture.get('depth', 0),
@@ -576,6 +591,7 @@ class LacusCore():
                 # We can give it another short.
                 raise RetryCapture(f'The capture of {url} has been cancelled.')
             except (TimeoutError, asyncio.exceptions.TimeoutError):
+                timeout_expired(capture_timeout, logger, 'Capture took too long.')
                 logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
                 result = {'error': f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)'}
                 raise CaptureError(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')

{lacuscore-1.9.3 → lacuscore-1.9.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.9.3"
+version = "1.9.4"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -29,12 +29,12 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
-requests = "^2.31.0"
+requests = "^2.32.1"
 Sphinx = [
    {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.24.6"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.24.10"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
 redis = {version = "^5.0.4", extras = ["hiredis"]}
@@ -47,14 +47,14 @@ docs = ["Sphinx"]
 [tool.poetry.group.dev.dependencies]
 types-redis = {version = "^4.6.0.20240425"}
 mypy = "^1.10.0"
-types-requests = "^2.31.0.20240406"
-types-beautifulsoup4 = "^4.12.0.20240229"
+types-requests = "^2.32.0.20240521"
+types-beautifulsoup4 = "^4.12.0.20240511"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-pytest = "^8.2.0"
+pytest = "^8.2.1"
 
 [build-system]
 requires = ["poetry_core"]