lacuscore 1.9.2__py3-none-any.whl → 1.9.4__py3-none-any.whl

lacuscore/__init__.py

@@ -1,4 +1,5 @@
-from .lacuscore import LacusCore, CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings  # noqa
+from .lacuscore import LacusCore
+from .helpers import CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings  # noqa
 from .lacus_monitoring import LacusCoreMonitoring  # noqa
 
 __all__ = [

lacuscore/helpers.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from enum import IntEnum, unique
+from logging import LoggerAdapter
+from typing import MutableMapping, Any, TypedDict
+
+from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
+
+
+class LacusCoreException(Exception):
+    pass
+
+
+class CaptureError(LacusCoreException):
+    pass
+
+
+class RetryCapture(LacusCoreException):
+    pass
+
+
+class CaptureSettingsError(LacusCoreException):
+    pass
+
+
+class LacusCoreLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
+    """
+    Prepend log entry with the UUID of the capture
+    """
+    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
+        if self.extra:
+            return '[{}] {}'.format(self.extra['uuid'], msg), kwargs
+        return msg, kwargs
+
+
+@unique
+class CaptureStatus(IntEnum):
+    '''The status of the capture'''
+    UNKNOWN = -1
+    QUEUED = 0
+    DONE = 1
+    ONGOING = 2
+
+
+class CaptureResponse(PlaywrightCaptureResponse, TypedDict, total=False):
+    '''A capture made by Lacus. With the base64 encoded image and downloaded file decoded to bytes.'''
+
+    # Need to make sure the type is what's expected down the line
+    children: list[CaptureResponse] | None  # type: ignore[misc]
+
+    status: int
+    runtime: float | None
+
+
+class CaptureResponseJson(TypedDict, total=False):
+    '''A capture made by Lacus. With the base64 encoded image and downloaded file *not* decoded.'''
+
+    status: int
+    last_redirected_url: str | None
+    har: dict[str, Any] | None
+    cookies: list[dict[str, str]] | None
+    error: str | None
+    html: str | None
+    png: str | None
+    downloaded_filename: str | None
+    downloaded_file: str | None
+    children: list[CaptureResponseJson] | None
+    runtime: float | None
+    potential_favicons: list[str] | None
+
+
+class CaptureSettings(TypedDict, total=False):
+    '''The capture settings that can be passed to Lacus.'''
+
+    url: str | None
+    document_name: str | None
+    document: str | None
+    browser: str | None
+    device_name: str | None
+    user_agent: str | None
+    proxy: str | dict[str, str] | None
+    general_timeout_in_sec: int | None
+    cookies: list[dict[str, Any]] | None
+    headers: str | dict[str, str] | None
+    http_credentials: dict[str, str] | None
+    geolocation: dict[str, float] | None
+    timezone_id: str | None
+    locale: str | None
+    color_scheme: str | None
+    viewport: dict[str, int] | None
+    referer: str | None
+    with_favicon: bool
+    allow_tracking: bool
+    force: bool
+    recapture_interval: int
+    priority: int
+    uuid: str | None
+
+    depth: int
+    rendered_hostname_only: bool  # Note: only used if depth is > 0

lacuscore/lacuscore.py

@@ -18,12 +18,10 @@ import zlib
 from asyncio import Task
 from base64 import b64decode, b64encode
 from datetime import date, timedelta
-from enum import IntEnum, unique
 from ipaddress import ip_address, IPv4Address, IPv6Address
-from logging import LoggerAdapter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Literal, Any, TypedDict, overload, cast, MutableMapping, Iterator
+from typing import Literal, Any, overload, cast, Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
@@ -33,17 +31,31 @@ from dns.exception import Timeout as DNSTimeout
 
 from defang import refang  # type: ignore[import-untyped]
 from playwrightcapture import Capture, PlaywrightCaptureException
-from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import DataError
 from ua_parser import user_agent_parser  # type: ignore[import-untyped]
 
+from . import task_logger
+from .helpers import (
+    LacusCoreLogAdapter, CaptureError, RetryCapture, CaptureSettingsError,
+    CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings)
+
 if sys.version_info < (3, 11):
     from async_timeout import timeout
+
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired:
+            logger.warning(f'Timeout expired: {error_message}')
+
 else:
     from asyncio import timeout
 
+    def timeout_expired(timeout_cm, logger, error_message: str) -> None:  # type: ignore[no-untyped-def]
+        if timeout_cm.expired():
+            logger.warning(f'Timeout expired: {error_message}')
+
+
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
 
@@ -65,95 +77,6 @@ def _secure_filename(filename: str) -> str:
     return filename
 
 
-class LacusCoreException(Exception):
-    pass
-
-
-class CaptureError(LacusCoreException):
-    pass
-
-
-class RetryCapture(LacusCoreException):
-    pass
-
-
-@unique
-class CaptureStatus(IntEnum):
-    '''The status of the capture'''
-    UNKNOWN = -1
-    QUEUED = 0
-    DONE = 1
-    ONGOING = 2
-
-
-class CaptureResponse(PlaywrightCaptureResponse, TypedDict, total=False):
-    '''A capture made by Lacus. With the base64 encoded image and downloaded file decoded to bytes.'''
-
-    # Need to make sure the type is what's expected down the line
-    children: list[CaptureResponse] | None  # type: ignore[misc]
-
-    status: int
-    runtime: float | None
-
-
-class CaptureResponseJson(TypedDict, total=False):
-    '''A capture made by Lacus. With the base64 encoded image and downloaded file *not* decoded.'''
-
-    status: int
-    last_redirected_url: str | None
-    har: dict[str, Any] | None
-    cookies: list[dict[str, str]] | None
-    error: str | None
-    html: str | None
-    png: str | None
-    downloaded_filename: str | None
-    downloaded_file: str | None
-    children: list[CaptureResponseJson] | None
-    runtime: float | None
-    potential_favicons: list[str] | None
-
-
-class CaptureSettings(TypedDict, total=False):
-    '''The capture settings that can be passed to Lacus.'''
-
-    url: str | None
-    document_name: str | None
-    document: str | None
-    browser: str | None
-    device_name: str | None
-    user_agent: str | None
-    proxy: str | dict[str, str] | None
-    general_timeout_in_sec: int | None
-    cookies: list[dict[str, Any]] | None
-    headers: str | dict[str, str] | None
-    http_credentials: dict[str, str] | None
-    geolocation: dict[str, float] | None
-    timezone_id: str | None
-    locale: str | None
-    color_scheme: str | None
-    viewport: dict[str, int] | None
-    referer: str | None
-    with_favicon: bool
-    allow_tracking: bool
-    force: bool
-    recapture_interval: int
-    priority: int
-    uuid: str | None
-
-    depth: int
-    rendered_hostname_only: bool  # Note: only used if depth is > 0
-
-
-class LacusCoreLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
-    """
-    Prepend log entry with the UUID of the capture
-    """
-    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
-        if self.extra:
-            return '[{}] {}'.format(self.extra['uuid'], msg), kwargs
-        return msg, kwargs
-
-
 class LacusCore():
     """Capture URLs or web enabled documents using PlaywrightCapture.
 
@@ -368,7 +291,7 @@ class LacusCore():
             p.execute()
         except DataError:
             self.master_logger.exception(f'Unable to enqueue: {to_enqueue}')
-            raise LacusCoreException(f'Unable to enqueue: {to_enqueue}')
+            raise CaptureSettingsError(f'Unable to enqueue: {to_enqueue}')
         return perma_uuid
 
     def _encode_response(self, capture: CaptureResponse) -> CaptureResponseJson:
@@ -453,7 +376,10 @@ class LacusCore():
             max_consume -= 1
             uuid: str = value[0][0].decode()
             priority: int = int(value[0][1])
-            yield asyncio.create_task(self._capture(uuid, priority), name=uuid)
+            logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid})
+            yield task_logger.create_task(self._capture(uuid, priority), name=uuid,
+                                          logger=logger,
+                                          message='Capture raised an uncaught exception')
 
     async def _capture(self, uuid: str, priority: int) -> None:
         """Trigger a specific capture
@@ -508,11 +434,11 @@ class LacusCore():
                     elif k == 'document':
                         document_as_bytes = b64decode(v)
                     else:
-                        raise LacusCoreException(f'Unexpected setting: {k}: {v}')
-            except LacusCoreException as e:
+                        raise CaptureSettingsError(f'Unexpected setting: {k}: {v}')
+            except CaptureSettingsError as e:
                 raise e
             except Exception as e:
-                raise LacusCoreException(f'Error while preparing settings: {e}')
+                raise CaptureSettingsError(f'Error while preparing settings: {e}')
 
             if not to_capture:
                 all_entries = self.redis.hgetall(f'lacus:capture_settings:{uuid}')
@@ -523,7 +449,7 @@ class LacusCore():
                 # we do not have a URL yet.
                 name = to_capture.pop('document_name', None)
                 if not name:
-                    raise LacusCoreException('No document name provided, settings are invalid')
+                    raise CaptureSettingsError('No document name provided, settings are invalid')
                 if not Path(name).suffix:
                     # The browser will simply display the file as text if there is no extension.
                     # Just add HTML as a fallback, as it will be the most comon one.
@@ -607,8 +533,6 @@ class LacusCore():
                     browser_engine = 'webkit'
             try:
                 logger.debug(f'Capturing {url}')
-                # NOTE: starting with python 3.11, we can use asyncio.timeout
-                # async with asyncio.timeout(self.max_capture_time):
                 general_timeout = to_capture.get('general_timeout_in_sec')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
@@ -629,12 +553,19 @@ class LacusCore():
                     capture.locale = to_capture.get('locale')  # type: ignore[assignment]
                     capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
                     try:
-                        async with timeout(general_timeout):
+                        # make sure the initialization doesn't take too long
+                        if general_timeout is None:
+                            general_timeout = 5
+                        init_timeout = max(general_timeout / 2, 5)
+                        async with timeout(init_timeout) as initialize_timeout:
                             await capture.initialize_context()
+
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
+                        timeout_expired(initialize_timeout, logger, 'Initializing took too long.')
                         logger.warning(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
                         raise RetryCapture(f'Initializing the context for {url} took longer than the allowed general timeout ({general_timeout}s)')
-                    async with timeout(self.max_capture_time):
+
+                    async with timeout(self.max_capture_time) as capture_timeout:
                         playwright_result = await capture.capture_page(
                             url, referer=to_capture.get('referer'),
                             depth=to_capture.get('depth', 0),
@@ -660,6 +591,7 @@ class LacusCore():
                 # We can give it another short.
                 raise RetryCapture(f'The capture of {url} has been cancelled.')
             except (TimeoutError, asyncio.exceptions.TimeoutError):
+                timeout_expired(capture_timeout, logger, 'Capture took too long.')
                 logger.warning(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
                 result = {'error': f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)'}
                 raise CaptureError(f'The capture of {url} took longer than the allowed max capture time ({self.max_capture_time}s)')
@@ -688,7 +620,7 @@ class LacusCore():
             else:
                 current_retry = int(_current_retry.decode())
                 if current_retry > 0:
-                    logger.debug(f'Retrying {url} for the {self.max_retries-current_retry+1}th time.')
+                    logger.debug(f'Retrying {url} for the {self.max_retries - current_retry + 1}th time.')
                     self.redis.decr(f'lacus:capture_retry:{uuid}')
                     retry = True
                 else:

lacuscore/task_logger.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from typing import Any, Coroutine, Optional, TypeVar, Tuple
+
+import asyncio
+import functools
+import logging
+
+from .helpers import LacusCoreLogAdapter
+
+T = TypeVar('T')
+
+# Code from https://quantlane.com/blog/ensure-asyncio-task-exceptions-get-logged/
+
+
+def create_task(
+    coroutine: Coroutine[Any, Any, T],
+    *,
+    name: str,
+    logger: 'LacusCoreLogAdapter',
+    message: str,
+    message_args: Tuple[Any, ...] = (),
+    loop: Optional[asyncio.AbstractEventLoop] = None,
+
+) -> 'asyncio.Task[T]':  # This type annotation has to be quoted for Python < 3.9, see https://www.python.org/dev/peps/pep-0585/
+    '''
+    This helper function wraps a ``loop.create_task(coroutine())`` call and ensures there is
+    an exception handler added to the resulting task. If the task raises an exception it is logged
+    using the provided ``logger``, with additional context provided by ``message`` and optionally
+    ``message_args``.
+    '''
+    if loop is None:
+        loop = asyncio.get_running_loop()
+    task = loop.create_task(coroutine, name=name)
+    task.add_done_callback(
+        functools.partial(_handle_task_result, logger=logger, message=message, message_args=message_args)
+    )
+    return task
+
+
+def _handle_task_result(
+    task: asyncio.Task[Any],
+    *,
+    logger: logging.Logger,
+    message: str,
+    message_args: Tuple[Any, ...] = (),
+) -> None:
+    try:
+        task.result()
+    except asyncio.CancelledError:
+        pass  # Task cancellation should not be logged as an error.
+    except asyncio.TimeoutError:
+        pass  # Timeout is also fine
+    # Ad the pylint ignore: we want to handle all exceptions here so that the result of the task
+    # is properly logged. There is no point re-raising the exception in this callback.
+    except Exception:  # pylint: disable=broad-except
+        logger.exception(message, *message_args)

{lacuscore-1.9.2.dist-info → lacuscore-1.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.2
+Version: 1.9.4
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,9 +28,9 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.5,<2.0.0)
-Requires-Dist: redis[hiredis] (>=5.0.3,<6.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.10,<2.0.0)
+Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
+Requires-Dist: requests (>=2.32.1,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/ail-project/LacusCore

lacuscore-1.9.4.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=hM4lKoPNybDCUMWdXTVVI1gRk_riLvRZ7IwFbamZLzE,341
+lacuscore/helpers.py,sha256=lULN7HhY-4a4HG-ybIt4jO3wEGTxkm_jKNqsGpNZo4Y,2711
+lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
+lacuscore/lacuscore.py,sha256=zi6oOcwmZ0yENIfJy55baeGXw3Js4lg5fmQz0YFRNJk,43030
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=8WbdJdKnGeFCxt9gtCNLI9vAQQZbsy2I5PRQpHP7XFU,1916
+lacuscore-1.9.4.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.9.4.dist-info/METADATA,sha256=Lr1wGWp4zfC19UwS51sjTxznPywLBZ7-0e9rAF88cYs,2629
+lacuscore-1.9.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lacuscore-1.9.4.dist-info/RECORD,,

lacuscore-1.9.2.dist-info/RECORD

@@ -1,8 +0,0 @@
-lacuscore/__init__.py,sha256=10JsUmjLD9xQIsjTDYT5nGowsJVe9C-FHNkaUFuAUtU,321
-lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
-lacuscore/lacuscore.py,sha256=2af-b9vy54FSlcY_YbfVcxsnU1AaeOjGi4UBaeumizs,44432
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore-1.9.2.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.9.2.dist-info/METADATA,sha256=3wrhwQf3_NnZ7FEZZjluPoUaK8d0FEzktfgzgpXi6-0,2628
-lacuscore-1.9.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lacuscore-1.9.2.dist-info/RECORD,,