lacuscore 1.9.4__py3-none-any.whl → 1.9.6__py3-none-any.whl

lacuscore/lacuscore.py

@@ -531,20 +531,40 @@ class LacusCore():
                     browser_engine = 'firefox'
                 else:
                     browser_engine = 'webkit'
+
+            cookies: list[dict[str, Any]] = []
+            if to_capture.get('cookies') and to_capture['cookies'] is not None:
+                # In order to properly pass the cookies to playwright,
+                # each of then must have a name, a value and either a domain + path or a URL
+                # Name and value are mandatory, and we cannot auto-fill them.
+                # If the cookie doesn't have a domain + path OR a URL, we fill the domain
+                # with the hostname of the URL we try to capture and the path with "/"
+                for cookie in to_capture['cookies']:
+                    if len(cookie) == 1:
+                        # we have a cookie in the format key: value
+                        name, value = cookie.popitem()
+                        cookie = {'name': name, 'value': value}
+                    if 'name' not in cookie or 'value' not in cookie:
+                        logger.warning(f'Invalid cookie: {cookie}')
+                        continue
+                    if 'domain' not in cookie and 'url' not in cookie:
+                        cookie['domain'] = splitted_url.hostname
+                        cookie['path'] = '/'
+                    cookies.append(cookie)
+
             try:
                 logger.debug(f'Capturing {url}')
-                general_timeout = to_capture.get('general_timeout_in_sec')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
                         browser=browser_engine,
                         device_name=to_capture.get('device_name'),
                         proxy=proxy,
-                        general_timeout_in_sec=general_timeout,
+                        general_timeout_in_sec=to_capture.get('general_timeout_in_sec'),
                         loglevel=self.master_logger.getEffectiveLevel(),
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.get('headers')  # type: ignore[assignment]
-                    capture.cookies = to_capture.get('cookies')  # type: ignore[assignment]
+                    capture.cookies = cookies  # type: ignore[assignment]
                     capture.viewport = to_capture.get('viewport')  # type: ignore[assignment]
                     capture.user_agent = to_capture.get('user_agent')  # type: ignore[assignment]
                     capture.http_credentials = to_capture.get('http_credentials')  # type: ignore[assignment]
@@ -552,27 +572,30 @@ class LacusCore():
                     capture.timezone_id = to_capture.get('timezone_id')  # type: ignore[assignment]
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
+
+                    # make sure the initialization doesn't take too long
+                    init_timeout = max(self.max_capture_time / 10, 5)
                     try:
-                        # make sure the initialization doesn't take too long
-                        if general_timeout is None:
-                            general_timeout = 5
-                        init_timeout = max(general_timeout / 2, 5)
                         async with timeout(init_timeout) as initialize_timeout:
                             await capture.initialize_context()
-
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
                         timeout_expired(initialize_timeout, logger, 'Initializing took too long.')
-                        logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                        raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-
-                    async with timeout(self.max_capture_time) as capture_timeout:
-                        playwright_result = await capture.capture_page(
-                            url, referer=to_capture.get('referer'),
-                            depth=to_capture.get('depth', 0),
-                            rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
-                            with_favicon=to_capture.get('with_favicon', False),
-                            allow_tracking=to_capture.get('allow_tracking', False),
-                            max_depth_capture_time=self.max_capture_time)
+                        logger.warning(f'Initializing the context for {url} took longer than the allowed initialization timeout ({init_timeout}s)')
+                        raise RetryCapture(f'Initializing the context for {url} took longer than the allowed initialization timeout ({init_timeout}s)')
+
+                    try:
+                        async with timeout(self.max_capture_time) as capture_timeout:
+                            playwright_result = await capture.capture_page(
+                                url, referer=to_capture.get('referer'),
+                                depth=to_capture.get('depth', 0),
+                                rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
+                                with_favicon=to_capture.get('with_favicon', False),
+                                allow_tracking=to_capture.get('allow_tracking', False),
+                                max_depth_capture_time=self.max_capture_time)
+                    except (TimeoutError, asyncio.exceptions.TimeoutError):
+                        timeout_expired(capture_timeout, logger, 'Capture took too long.')
+                        logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
+                        raise RetryCapture(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
                     result = cast(CaptureResponse, playwright_result)
                     if 'error' in result and 'error_name' in result:
                         # generate stats
@@ -581,7 +604,7 @@ class LacusCore():
             except RetryCapture as e:
                 raise e
             except PlaywrightCaptureException as e:
-                logger.exception(f'Invalid parameters for the capture of {url} - {e}')
+                logger.warning(f'Invalid parameters for the capture of {url} - {e}')
                 result = {'error': f'Invalid parameters for the capture of {url} - {e}'}
                 raise CaptureError(f'Invalid parameters for the capture of {url} - {e}')
             except asyncio.CancelledError:
@@ -590,11 +613,6 @@ class LacusCore():
                 # The capture can be canceled if it has been running for way too long.
                 # We can give it another short.
                 raise RetryCapture(f'The capture of {url} has been cancelled.')
-            except (TimeoutError, asyncio.exceptions.TimeoutError):
-                timeout_expired(capture_timeout, logger, 'Capture took too long.')
-                logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
-                result = {'error': f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)'}
-                raise CaptureError(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
             except Exception as e:
                 logger.exception(f'Something went poorly {url} - {e}')
                 result = {'error': f'Something went poorly {url} - {e}'}

{lacuscore-1.9.4.dist-info → lacuscore-1.9.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.4
+Version: 1.9.6
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,9 +28,9 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.10,<2.0.0)
-Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
-Requires-Dist: requests (>=2.32.1,<3.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.11,<2.0.0)
+Requires-Dist: redis[hiredis] (>=5.0.6,<6.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/ail-project/LacusCore

{lacuscore-1.9.4.dist-info → lacuscore-1.9.6.dist-info}/RECORD

@@ -1,10 +1,10 @@
 lacuscore/__init__.py,sha256=hM4lKoPNybDCUMWdXTVVI1gRk_riLvRZ7IwFbamZLzE,341
 lacuscore/helpers.py,sha256=lULN7HhY-4a4HG-ybIt4jO3wEGTxkm_jKNqsGpNZo4Y,2711
 lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
-lacuscore/lacuscore.py,sha256=zi6oOcwmZ0yENIfJy55baeGXw3Js4lg5fmQz0YFRNJk,43030
+lacuscore/lacuscore.py,sha256=pk9E2w2zEH8l6VywXW4Ur_-XAMhFVDWTWIt2bIZ07Bo,44047
 lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lacuscore/task_logger.py,sha256=8WbdJdKnGeFCxt9gtCNLI9vAQQZbsy2I5PRQpHP7XFU,1916
-lacuscore-1.9.4.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.9.4.dist-info/METADATA,sha256=Lr1wGWp4zfC19UwS51sjTxznPywLBZ7-0e9rAF88cYs,2629
-lacuscore-1.9.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lacuscore-1.9.4.dist-info/RECORD,,
+lacuscore-1.9.6.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.9.6.dist-info/METADATA,sha256=HGVQl2vRm95AmDm9d0z0OU89GX-8ZGUDsiSR8eotx3Y,2629
+lacuscore-1.9.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lacuscore-1.9.6.dist-info/RECORD,,