lacuscore 1.9.2__tar.gz → 1.9.3__tar.gz

{lacuscore-1.9.2 → lacuscore-1.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.9.2
+Version: 1.9.3
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,8 +28,8 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.24.5,<2.0.0)
-Requires-Dist: redis[hiredis] (>=5.0.3,<6.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.24.6,<2.0.0)
+Requires-Dist: redis[hiredis] (>=5.0.4,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/

{lacuscore-1.9.2 → lacuscore-1.9.3}/lacuscore/__init__.py

@@ -1,4 +1,5 @@
-from .lacuscore import LacusCore, CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings  # noqa
+from .lacuscore import LacusCore
+from .helpers import CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings  # noqa
 from .lacus_monitoring import LacusCoreMonitoring  # noqa
 
 __all__ = [

lacuscore-1.9.3/lacuscore/helpers.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from enum import IntEnum, unique
+from logging import LoggerAdapter
+from typing import MutableMapping, Any, TypedDict
+
+from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
+
+
+class LacusCoreException(Exception):
+    pass
+
+
+class CaptureError(LacusCoreException):
+    pass
+
+
+class RetryCapture(LacusCoreException):
+    pass
+
+
+class CaptureSettingsError(LacusCoreException):
+    pass
+
+
+class LacusCoreLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
+    """
+    Prepend log entry with the UUID of the capture
+    """
+    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
+        if self.extra:
+            return '[{}] {}'.format(self.extra['uuid'], msg), kwargs
+        return msg, kwargs
+
+
+@unique
+class CaptureStatus(IntEnum):
+    '''The status of the capture'''
+    UNKNOWN = -1
+    QUEUED = 0
+    DONE = 1
+    ONGOING = 2
+
+
+class CaptureResponse(PlaywrightCaptureResponse, TypedDict, total=False):
+    '''A capture made by Lacus. With the base64 encoded image and downloaded file decoded to bytes.'''
+
+    # Need to make sure the type is what's expected down the line
+    children: list[CaptureResponse] | None  # type: ignore[misc]
+
+    status: int
+    runtime: float | None
+
+
+class CaptureResponseJson(TypedDict, total=False):
+    '''A capture made by Lacus. With the base64 encoded image and downloaded file *not* decoded.'''
+
+    status: int
+    last_redirected_url: str | None
+    har: dict[str, Any] | None
+    cookies: list[dict[str, str]] | None
+    error: str | None
+    html: str | None
+    png: str | None
+    downloaded_filename: str | None
+    downloaded_file: str | None
+    children: list[CaptureResponseJson] | None
+    runtime: float | None
+    potential_favicons: list[str] | None
+
+
+class CaptureSettings(TypedDict, total=False):
+    '''The capture settings that can be passed to Lacus.'''
+
+    url: str | None
+    document_name: str | None
+    document: str | None
+    browser: str | None
+    device_name: str | None
+    user_agent: str | None
+    proxy: str | dict[str, str] | None
+    general_timeout_in_sec: int | None
+    cookies: list[dict[str, Any]] | None
+    headers: str | dict[str, str] | None
+    http_credentials: dict[str, str] | None
+    geolocation: dict[str, float] | None
+    timezone_id: str | None
+    locale: str | None
+    color_scheme: str | None
+    viewport: dict[str, int] | None
+    referer: str | None
+    with_favicon: bool
+    allow_tracking: bool
+    force: bool
+    recapture_interval: int
+    priority: int
+    uuid: str | None
+
+    depth: int
+    rendered_hostname_only: bool  # Note: only used if depth is > 0

{lacuscore-1.9.2 → lacuscore-1.9.3}/lacuscore/lacuscore.py

@@ -18,12 +18,10 @@ import zlib
 from asyncio import Task
 from base64 import b64decode, b64encode
 from datetime import date, timedelta
-from enum import IntEnum, unique
 from ipaddress import ip_address, IPv4Address, IPv6Address
-from logging import LoggerAdapter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Literal, Any, TypedDict, overload, cast, MutableMapping, Iterator
+from typing import Literal, Any, overload, cast, Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
@@ -33,12 +31,16 @@ from dns.exception import Timeout as DNSTimeout
 
 from defang import refang  # type: ignore[import-untyped]
 from playwrightcapture import Capture, PlaywrightCaptureException
-from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import DataError
 from ua_parser import user_agent_parser  # type: ignore[import-untyped]
 
+from . import task_logger
+from .helpers import (
+    LacusCoreLogAdapter, CaptureError, RetryCapture, CaptureSettingsError,
+    CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings)
+
 if sys.version_info < (3, 11):
     from async_timeout import timeout
 else:
@@ -65,95 +67,6 @@ def _secure_filename(filename: str) -> str:
     return filename
 
 
-class LacusCoreException(Exception):
-    pass
-
-
-class CaptureError(LacusCoreException):
-    pass
-
-
-class RetryCapture(LacusCoreException):
-    pass
-
-
-@unique
-class CaptureStatus(IntEnum):
-    '''The status of the capture'''
-    UNKNOWN = -1
-    QUEUED = 0
-    DONE = 1
-    ONGOING = 2
-
-
-class CaptureResponse(PlaywrightCaptureResponse, TypedDict, total=False):
-    '''A capture made by Lacus. With the base64 encoded image and downloaded file decoded to bytes.'''
-
-    # Need to make sure the type is what's expected down the line
-    children: list[CaptureResponse] | None  # type: ignore[misc]
-
-    status: int
-    runtime: float | None
-
-
-class CaptureResponseJson(TypedDict, total=False):
-    '''A capture made by Lacus. With the base64 encoded image and downloaded file *not* decoded.'''
-
-    status: int
-    last_redirected_url: str | None
-    har: dict[str, Any] | None
-    cookies: list[dict[str, str]] | None
-    error: str | None
-    html: str | None
-    png: str | None
-    downloaded_filename: str | None
-    downloaded_file: str | None
-    children: list[CaptureResponseJson] | None
-    runtime: float | None
-    potential_favicons: list[str] | None
-
-
-class CaptureSettings(TypedDict, total=False):
-    '''The capture settings that can be passed to Lacus.'''
-
-    url: str | None
-    document_name: str | None
-    document: str | None
-    browser: str | None
-    device_name: str | None
-    user_agent: str | None
-    proxy: str | dict[str, str] | None
-    general_timeout_in_sec: int | None
-    cookies: list[dict[str, Any]] | None
-    headers: str | dict[str, str] | None
-    http_credentials: dict[str, str] | None
-    geolocation: dict[str, float] | None
-    timezone_id: str | None
-    locale: str | None
-    color_scheme: str | None
-    viewport: dict[str, int] | None
-    referer: str | None
-    with_favicon: bool
-    allow_tracking: bool
-    force: bool
-    recapture_interval: int
-    priority: int
-    uuid: str | None
-
-    depth: int
-    rendered_hostname_only: bool  # Note: only used if depth is > 0
-
-
-class LacusCoreLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
-    """
-    Prepend log entry with the UUID of the capture
-    """
-    def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
-        if self.extra:
-            return '[{}] {}'.format(self.extra['uuid'], msg), kwargs
-        return msg, kwargs
-
-
 class LacusCore():
     """Capture URLs or web enabled documents using PlaywrightCapture.
 
@@ -368,7 +281,7 @@ class LacusCore():
             p.execute()
         except DataError:
             self.master_logger.exception(f'Unable to enqueue: {to_enqueue}')
-            raise LacusCoreException(f'Unable to enqueue: {to_enqueue}')
+            raise CaptureSettingsError(f'Unable to enqueue: {to_enqueue}')
         return perma_uuid
 
     def _encode_response(self, capture: CaptureResponse) -> CaptureResponseJson:
@@ -453,7 +366,10 @@ class LacusCore():
             max_consume -= 1
             uuid: str = value[0][0].decode()
             priority: int = int(value[0][1])
-            yield asyncio.create_task(self._capture(uuid, priority), name=uuid)
+            logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid})
+            yield task_logger.create_task(self._capture(uuid, priority), name=uuid,
+                                          logger=logger,
+                                          message='Capture raised an uncaught exception')
 
     async def _capture(self, uuid: str, priority: int) -> None:
         """Trigger a specific capture
@@ -508,11 +424,11 @@ class LacusCore():
                     elif k == 'document':
                         document_as_bytes = b64decode(v)
                     else:
-                        raise LacusCoreException(f'Unexpected setting: {k}: {v}')
-            except LacusCoreException as e:
+                        raise CaptureSettingsError(f'Unexpected setting: {k}: {v}')
+            except CaptureSettingsError as e:
                 raise e
             except Exception as e:
-                raise LacusCoreException(f'Error while preparing settings: {e}')
+                raise CaptureSettingsError(f'Error while preparing settings: {e}')
 
             if not to_capture:
                 all_entries = self.redis.hgetall(f'lacus:capture_settings:{uuid}')
@@ -523,7 +439,7 @@ class LacusCore():
                 # we do not have a URL yet.
                 name = to_capture.pop('document_name', None)
                 if not name:
-                    raise LacusCoreException('No document name provided, settings are invalid')
+                    raise CaptureSettingsError('No document name provided, settings are invalid')
                 if not Path(name).suffix:
                     # The browser will simply display the file as text if there is no extension.
                     # Just add HTML as a fallback, as it will be the most comon one.
@@ -688,7 +604,7 @@ class LacusCore():
             else:
                 current_retry = int(_current_retry.decode())
                 if current_retry > 0:
-                    logger.debug(f'Retrying {url} for the {self.max_retries-current_retry+1}th time.')
+                    logger.debug(f'Retrying {url} for the {self.max_retries - current_retry + 1}th time.')
                     self.redis.decr(f'lacus:capture_retry:{uuid}')
                     retry = True
                 else:

lacuscore-1.9.3/lacuscore/task_logger.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from typing import Any, Coroutine, Optional, TypeVar, Tuple
+
+import asyncio
+import functools
+import logging
+
+from .helpers import LacusCoreLogAdapter
+
+T = TypeVar('T')
+
+# Code from https://quantlane.com/blog/ensure-asyncio-task-exceptions-get-logged/
+
+
+def create_task(
+    coroutine: Coroutine[Any, Any, T],
+    *,
+    name: str,
+    logger: 'LacusCoreLogAdapter',
+    message: str,
+    message_args: Tuple[Any, ...] = (),
+    loop: Optional[asyncio.AbstractEventLoop] = None,
+
+) -> 'asyncio.Task[T]':  # This type annotation has to be quoted for Python < 3.9, see https://www.python.org/dev/peps/pep-0585/
+    '''
+    This helper function wraps a ``loop.create_task(coroutine())`` call and ensures there is
+    an exception handler added to the resulting task. If the task raises an exception it is logged
+    using the provided ``logger``, with additional context provided by ``message`` and optionally
+    ``message_args``.
+    '''
+    if loop is None:
+        loop = asyncio.get_running_loop()
+    task = loop.create_task(coroutine, name=name)
+    task.add_done_callback(
+        functools.partial(_handle_task_result, logger=logger, message=message, message_args=message_args)
+    )
+    return task
+
+
+def _handle_task_result(
+    task: asyncio.Task[Any],
+    *,
+    logger: logging.Logger,
+    message: str,
+    message_args: Tuple[Any, ...] = (),
+) -> None:
+    try:
+        task.result()
+    except asyncio.CancelledError:
+        pass  # Task cancellation should not be logged as an error.
+    except asyncio.TimeoutError:
+        pass  # Timeout is also fine
+    # Ad the pylint ignore: we want to handle all exceptions here so that the result of the task
+    # is properly logged. There is no point re-raising the exception in this callback.
+    except Exception:  # pylint: disable=broad-except
+        logger.exception(message, *message_args)

{lacuscore-1.9.2 → lacuscore-1.9.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.9.2"
+version = "1.9.3"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -34,10 +34,10 @@ Sphinx = [
    {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.24.5"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.24.6"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
-redis = {version = "^5.0.3", extras = ["hiredis"]}
+redis = {version = "^5.0.4", extras = ["hiredis"]}
 dnspython = "^2.6.1"
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 
@@ -45,8 +45,8 @@ async-timeout = {version = "^4.0.3", python = "<3.11"}
 docs = ["Sphinx"]
 
 [tool.poetry.group.dev.dependencies]
-types-redis = {version = "^4.6.0.20240417"}
-mypy = "^1.9.0"
+types-redis = {version = "^4.6.0.20240425"}
+mypy = "^1.10.0"
 types-requests = "^2.31.0.20240406"
 types-beautifulsoup4 = "^4.12.0.20240229"
 ipython = [
@@ -54,7 +54,7 @@ ipython = [
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-pytest = "^8.1.1"
+pytest = "^8.2.0"
 
 [build-system]
 requires = ["poetry_core"]