lacuscore 1.9.3__py3-none-any.whl → 1.9.4__py3-none-any.whl

lacuscore/lacuscore.py

@@ -43,9 +43,19 @@ from .helpers import (
 
 if sys.version_info < (3, 11):
     from async_timeout import timeout
+
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired:
+            logger.warning(f'Timeout expired: {error_message}')
+
 else:
     from asyncio import timeout
 
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired():
+            logger.warning(f'Timeout expired: {error_message}')
+
+
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -523,8 +533,6 @@ class LacusCore():
                     browser_engine = 'webkit'
             try:
                 logger.debug(f'Capturing {url}')
-                # NOTE: starting with python 3.11, we can use asyncio.timeout
-                # async with asyncio.timeout(self.max_capture_time):
                 general_timeout = to_capture.get('general_timeout_in_sec')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
@@ -545,12 +553,19 @@ class LacusCore():
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
                     try:
-                        async with timeout(general_timeout):
+                        # make sure the initialization doesn't take too long
+                        if general_timeout is None:
+                            general_timeout = 5
+                        init_timeout = max(general_timeout / 2, 5)
+                        async with timeout(init_timeout) as initialize_timeout:
                             await capture.initialize_context()
+
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
+                        timeout_expired(initialize_timeout, logger, 'Initializing took too long.')
                         logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
                         raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                    async with timeout(self.max_capture_time):
+
+                    async with timeout(self.max_capture_time) as capture_timeout:
                         playwright_result = await capture.capture_page(
                             url, referer=to_capture.get('referer'),
                             depth=to_capture.get('depth', 0),
@@ -576,6 +591,7 @@ class LacusCore():
                 # We can give it another short.
                 raise RetryCapture(f'The capture of {url} has been cancelled.')
             except (TimeoutError, asyncio.exceptions.TimeoutError):
+                timeout_expired(capture_timeout, logger, 'Capture took too long.')
                 logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
                 result = {'error': f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)'}
                 raise CaptureError(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')

{lacuscore-1.9.3.dist-info → lacuscore-1.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.3
+Version: 1.9.4
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,9 +28,9 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.6,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.10,<2.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: requests (>=2.32.1,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/ail-project/LacusCore

{lacuscore-1.9.3.dist-info → lacuscore-1.9.4.dist-info}/RECORD

@@ -1,10 +1,10 @@
 lacuscore/__init__.py,sha256=hM4lKoPNybDCUMWdXTVVI1gRk_riLvRZ7IwFbamZLzE,341
 lacuscore/helpers.py,sha256=lULN7HhY-4a4HG-ybIt4jO3wEGTxkm_jKNqsGpNZo4Y,2711
 lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
-lacuscore/lacuscore.py,sha256=32ktK2AWse0US11SrrWFjCUsthFCTONnB52vfEhlvyY,42305
+lacuscore/lacuscore.py,sha256=zi6oOcwmZ0yENIfJy55baeGXw3Js4lg5fmQz0YFRNJk,43030
 lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lacuscore/task_logger.py,sha256=8WbdJdKnGeFCxt9gtCNLI9vAQQZbsy2I5PRQpHP7XFU,1916
-lacuscore-1.9.3.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.9.3.dist-info/METADATA,sha256=n7zZEUl94bN9pcPAK3xakRPJejNsjgUHa7aa_4fbJVk,2628
-lacuscore-1.9.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lacuscore-1.9.3.dist-info/RECORD,,
+lacuscore-1.9.4.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.9.4.dist-info/METADATA,sha256=Lr1wGWp4zfC19UwS51sjTxznPywLBZ7-0e9rAF88cYs,2629
+lacuscore-1.9.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lacuscore-1.9.4.dist-info/RECORD,,