lacuscore 1.10.0__py3-none-any.whl → 1.10.2__py3-none-any.whl

lacuscore/__init__.py

@@ -1,5 +1,6 @@
 from .lacuscore import LacusCore
-from .helpers import CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings  # noqa
+from .helpers import (CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings,  # noqa
+                      LacusCoreException, CaptureError, RetryCapture, CaptureSettingsError)  # noqa
 from .lacus_monitoring import LacusCoreMonitoring  # noqa
 
 __all__ = [
@@ -8,5 +9,9 @@ __all__ = [
     'CaptureResponse',
     'CaptureResponseJson',
     'CaptureSettings',
-    'LacusCoreMonitoring'
+    'LacusCoreMonitoring',
+    'LacusCoreException',
+    'CaptureError',
+    'RetryCapture',
+    'CaptureSettingsError'
 ]

lacuscore/helpers.py

@@ -2,9 +2,15 @@
 
 from __future__ import annotations
 
+import json
+
 from enum import IntEnum, unique
 from logging import LoggerAdapter
-from typing import MutableMapping, Any, TypedDict
+from typing import MutableMapping, Any, TypedDict, Literal, Mapping
+
+from defang import refang  # type: ignore[import-untyped]
+from pydantic import BaseModel, field_validator, model_validator, ValidationError
+from pydantic_core import from_json
 
 from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 
@@ -22,7 +28,11 @@ class RetryCapture(LacusCoreException):
 
 
 class CaptureSettingsError(LacusCoreException):
-    pass
+    '''Can handle Pydantic validation errors'''
+
+    def __init__(self, message: str, pydantic_validation_errors: ValidationError | None=None) -> None:
+        super().__init__(message)
+        self.pydantic_validation_errors = pydantic_validation_errors
 
 
 class LacusCoreLogAdapter(LoggerAdapter):  # type: ignore[type-arg]
@@ -71,32 +81,198 @@ class CaptureResponseJson(TypedDict, total=False):
     potential_favicons: list[str] | None
 
 
-class CaptureSettings(TypedDict, total=False):
+class CaptureSettings(BaseModel):
     '''The capture settings that can be passed to Lacus.'''
 
-    url: str | None
-    document_name: str | None
-    document: str | None
-    browser: str | None
-    device_name: str | None
-    user_agent: str | None
-    proxy: str | dict[str, str] | None
-    general_timeout_in_sec: int | None
-    cookies: list[dict[str, Any]] | None
-    headers: str | dict[str, str] | None
-    http_credentials: dict[str, str] | None
-    geolocation: dict[str, float] | None
-    timezone_id: str | None
-    locale: str | None
-    color_scheme: str | None
-    viewport: dict[str, int] | None
-    referer: str | None
-    with_favicon: bool
-    allow_tracking: bool
-    force: bool
-    recapture_interval: int
-    priority: int
-    uuid: str | None
-
-    depth: int
-    rendered_hostname_only: bool  # Note: only used if depth is > 0
+    url: str | None = None
+    document_name: str | None = None
+    document: str | None = None
+    browser: Literal['chromium', 'firefox', 'webkit'] | None = None
+    device_name: str | None = None
+    user_agent: str | None = None
+    proxy: str | dict[str, str] | None = None
+    general_timeout_in_sec: int | None = None
+    cookies: list[dict[str, Any]] | None = None
+    headers: dict[str, str] | None = None
+    http_credentials: dict[str, str] | None = None
+    geolocation: dict[str, float] | None = None
+    timezone_id: str | None = None
+    locale: str | None = None
+    color_scheme: str | None = None
+    viewport: dict[str, int] | None = None
+    referer: str | None = None
+    with_favicon: bool = False
+    allow_tracking: bool = False
+    force: bool = False
+    recapture_interval: int = 300
+    priority: int = 0
+    uuid: str | None = None
+
+    depth: int = 0
+    rendered_hostname_only: bool = True  # Note: only used if depth is > 0
+
+    @model_validator(mode='after')
+    def check_capture_element(self) -> CaptureSettings:
+        if self.document_name and not self.document:
+            raise CaptureSettingsError('You must provide a document if you provide a document name')
+        if self.document and not self.document_name:
+            raise CaptureSettingsError('You must provide a document name if you provide a document')
+
+        if self.url and (self.document or self.document_name):
+            raise CaptureSettingsError('You cannot provide both a URL and a document to capture')
+        if not self.url and not (self.document and self.document_name):
+            raise CaptureSettingsError('You must provide either a URL or a document to capture')
+        return self
+
+    @field_validator('url', mode='after')
+    @classmethod
+    def load_url(cls, v: str | None) -> str | None:
+        if isinstance(v, str):
+            url = v.strip()
+            url = refang(url)  # In case we get a defanged url at this stage.
+            if (not url.lower().startswith('data:')
+                    and not url.lower().startswith('http:')
+                    and not url.lower().startswith('https:')
+                    and not url.lower().startswith('file:')):
+                url = f'http://{url}'
+            return url
+        return v
+
+    @field_validator('document_name', mode='after')
+    @classmethod
+    def load_document_name(cls, v: str | None) -> str | None:
+        if isinstance(v, str):
+            name = v.strip()
+            if '.' not in name:
+                # The browser will simply display the file as text if there is no extension.
+                # Just add HTML as a fallback, as it will be the most comon one.
+                name = f'{name}.html'
+            return name
+        return v
+
+    @field_validator('proxy', mode='before')
+    @classmethod
+    def load_proxy_json(cls, v: Any) -> str | dict[str, str] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v.startswith('{'):
+                return from_json(v)
+            # Just the proxy
+            return v
+        elif isinstance(v, dict):
+            return v
+        return None
+
+    @field_validator('cookies', mode='before')
+    @classmethod
+    def load_cookies_json(cls, v: Any) -> list[dict[str, Any]] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v.startswith('['):
+                return from_json(v)
+            # Cookies are invalid, ignoring.
+        elif isinstance(v, list):
+            return v
+        return None
+
+    @field_validator('headers', mode='before')
+    @classmethod
+    def load_headers_json(cls, v: Any) -> dict[str, str] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v[0] == '{':
+                return from_json(v)
+            else:
+                # make it a dict
+                new_headers = {}
+                for header_line in v.splitlines():
+                    if header_line and ':' in header_line:
+                        splitted = header_line.split(':', 1)
+                        if splitted and len(splitted) == 2:
+                            header, h_value = splitted
+                            if header.strip() and h_value.strip():
+                                new_headers[header.strip()] = h_value.strip()
+                return new_headers
+        elif isinstance(v, dict):
+            return v
+        return None
+
+    @field_validator('http_credentials', mode='before')
+    @classmethod
+    def load_http_creds_json(cls, v: Any) -> dict[str, str] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v.startswith('{'):
+                return from_json(v)
+        elif isinstance(v, dict):
+            return v
+        return None
+
+    @field_validator('http_credentials', mode='after')
+    @classmethod
+    def check_http_creds(cls, v: dict[str, str] | None) -> dict[str, str] | None:
+        if not v:
+            return v
+        if 'username' in v and 'password' in v:
+            return v
+        raise CaptureSettingsError(f'HTTP credentials must have a username and a password: {v}')
+
+    @field_validator('geolocation', mode='before')
+    @classmethod
+    def load_geolocation_json(cls, v: Any) -> dict[str, float] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v.startswith('{'):
+                return from_json(v)
+        elif isinstance(v, dict):
+            return v
+        return None
+
+    @field_validator('geolocation', mode='after')
+    @classmethod
+    def check_geolocation(cls, v: dict[str, float] | None) -> dict[str, float] | None:
+        if not v:
+            return v
+        if 'latitude' in v and 'longitude' in v:
+            return v
+        raise CaptureSettingsError(f'A geolocation must have a latitude and a longitude: {v}')
+
+    @field_validator('viewport', mode='before')
+    @classmethod
+    def load_viewport_json(cls, v: Any) -> dict[str, int] | None:
+        if not v:
+            return None
+        if isinstance(v, str):
+            if v.startswith('{'):
+                return from_json(v)
+        elif isinstance(v, dict):
+            return v
+        return None
+
+    @field_validator('viewport', mode='after')
+    @classmethod
+    def check_viewport(cls, v: dict[str, int] | None) -> dict[str, int] | None:
+        if not v:
+            return v
+        if 'width' in v and 'height' in v:
+            return v
+        raise CaptureSettingsError(f'A viewport must have a width and a height: {v}')
+
+    def redis_dump(self) -> Mapping[str | bytes, bytes | float | int | str]:
+        mapping_capture: dict[str | bytes, bytes | float | int | str] = {}
+        for key, value in dict(self).items():
+            if value is None:
+                continue
+            if isinstance(value, bool):
+                mapping_capture[key] = 1 if value else 0
+            elif isinstance(value, (list, dict)):
+                if value:
+                    mapping_capture[key] = json.dumps(value)
+            elif isinstance(value, (bytes, float, int, str)) and value not in ['', b'']:  # we're ok with 0 for example
+                mapping_capture[key] = value
+        return mapping_capture

lacuscore/lacuscore.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import asyncio
 import hashlib
-import json
 import logging
 import os
 import pickle
@@ -19,7 +18,6 @@ from asyncio import Task
 from base64 import b64decode, b64encode
 from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
-from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Literal, Any, overload, cast, Iterator
 from uuid import uuid4
@@ -29,8 +27,8 @@ from dns import resolver
 from dns.exception import DNSException
 from dns.exception import Timeout as DNSTimeout
 
-from defang import refang  # type: ignore[import-untyped]
-from playwrightcapture import Capture, PlaywrightCaptureException
+from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter
+from pydantic import ValidationError
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import DataError
@@ -114,7 +112,7 @@ class LacusCore():
         return bool(self.redis.ping())
 
     @overload
-    def enqueue(self, *, settings: CaptureSettings | None=None) -> str:
+    def enqueue(self, *, settings: dict[str, Any] | None=None) -> str:
         ...
 
     @overload
@@ -127,7 +125,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
-                headers: str | dict[str, str] | None=None,
+                headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
                 timezone_id: str | None=None,
@@ -146,7 +144,7 @@ class LacusCore():
         ...
 
     def enqueue(self, *,
-                settings: CaptureSettings | None=None,
+                settings: dict[str, Any] | None=None,
                 url: str | None=None,
                 document_name: str | None=None, document: str | None=None,
                 depth: int=0,
@@ -155,7 +153,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
-                headers: str | dict[str, str] | None=None,
+                headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
                 timezone_id: str | None=None,
@@ -177,7 +175,7 @@ class LacusCore():
 
         :param url: URL to capture (incompatible with document and document_name)
         :param document_name: Filename of the document to capture (required if document is used)
-        :param document: Document to capture itself (requires a document_name)
+        :param document: Document to capture itself (requires a document_name), must be base64 encoded
         :param depth: [Dangerous] Depth of the capture. If > 0, the URLs of the rendered document will be extracted and captured. It can take a very long time.
         :param browser: The prowser to use for the capture
         :param device_name: The name of the device, must be something Playwright knows
@@ -203,56 +201,24 @@ class LacusCore():
 
         :return: UUID, reference to the capture for later use
         """
-        to_enqueue: CaptureSettings
-        if settings:
-            if 'url' in settings and settings['url'] is not None:
-                settings['url'] = settings['url'].strip()
-            if settings.get('force') is not None:
-                force = settings.pop('force', False)
-            if settings.get('recapture_interval') is not None:
-                recapture_interval = settings.pop('recapture_interval', 300)
-            if settings.get('priority') is not None:
-                priority = settings.pop('priority', 0)
-            to_enqueue = settings
-        else:
-            to_enqueue = {'depth': depth, 'rendered_hostname_only': rendered_hostname_only}
-            if url:
-                to_enqueue['url'] = url.strip()
-            elif document_name and document:
-                to_enqueue['document_name'] = _secure_filename(document_name)
-                to_enqueue['document'] = document
-            if browser:
-                to_enqueue['browser'] = browser
-            if device_name:
-                to_enqueue['device_name'] = device_name
-            if user_agent:
-                to_enqueue['user_agent'] = user_agent
-            if proxy:
-                to_enqueue['proxy'] = proxy
-            if general_timeout_in_sec is not None:  # that would be a terrible idea, but this one could be 0
-                to_enqueue['general_timeout_in_sec'] = general_timeout_in_sec
-            if cookies:
-                to_enqueue['cookies'] = cookies
-            if headers:
-                to_enqueue['headers'] = headers
-            if http_credentials:
-                to_enqueue['http_credentials'] = http_credentials
-            if geolocation:
-                to_enqueue['geolocation'] = geolocation
-            if timezone_id:
-                to_enqueue['timezone_id'] = timezone_id
-            if locale:
-                to_enqueue['locale'] = locale
-            if color_scheme:
-                to_enqueue['color_scheme'] = color_scheme
-            if viewport:
-                to_enqueue['viewport'] = viewport
-            if referer:
-                to_enqueue['referer'] = referer
-            if with_favicon:
-                to_enqueue['with_favicon'] = with_favicon
-            if allow_tracking:
-                to_enqueue['allow_tracking'] = allow_tracking
+        if not settings:
+            settings = {'depth': depth, 'rendered_hostname_only': rendered_hostname_only,
+                        'url': url, 'document_name': document_name, 'document': document,
+                        'browser': browser, 'device_name': device_name,
+                        'user_agent': user_agent, 'proxy': proxy,
+                        'general_timeout_in_sec': general_timeout_in_sec,
+                        'cookies': cookies, 'headers': headers,
+                        'http_credentials': http_credentials, 'geolocation': geolocation,
+                        'timezone_id': timezone_id, 'locale': locale,
+                        'color_scheme': color_scheme, 'viewport': viewport,
+                        'referer': referer, 'with_favicon': with_favicon,
+                        'allow_tracking': allow_tracking}
+
+        try:
+            to_enqueue = CaptureSettings(**settings)
+        except ValidationError as e:
+            self.master_logger.warning(f'Unable to validate settings: {e}.')
+            raise CaptureSettingsError('Invalid settings', e)
 
         hash_query = hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest()
         if not force:
@@ -271,21 +237,9 @@ class LacusCore():
         else:
             perma_uuid = str(uuid4())
 
-        mapping_capture: dict[str, bytes | float | int | str] = {}
-        for key, value in to_enqueue.items():
-            if value is None:
-                continue
-            if isinstance(value, bool):
-                mapping_capture[key] = 1 if value else 0
-            elif isinstance(value, (list, dict)):
-                if value:
-                    mapping_capture[key] = json.dumps(value)
-            elif isinstance(value, (bytes, float, int, str)) and value not in ['', b'']:  # we're ok with 0 for example
-                mapping_capture[key] = value
-
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
-        p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=mapping_capture)  # type: ignore[arg-type]
+        p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump())
         p.zadd('lacus:to_capture', {perma_uuid: priority if priority is not None else 0})
         try:
             p.execute()
@@ -399,77 +353,33 @@ class LacusCore():
 
         retry = False
         try:
-            setting_keys = ['depth', 'rendered_hostname_only', 'url', 'document_name',
-                            'document', 'browser', 'device_name', 'user_agent', 'proxy',
-                            'general_timeout_in_sec', 'cookies', 'headers', 'http_credentials',
-                            'viewport', 'referer', 'geolocation', 'timezone_id', 'locale',
-                            'color_scheme', 'with_favicon', 'allow_tracking']
             result: CaptureResponse = {}
-            to_capture: CaptureSettings = {}
-            document_as_bytes = b''
+            _to_capture: dict[bytes, Any] = {}
             url: str = ''
-            try:
-                for k, v in zip(setting_keys, self.redis.hmget(f'lacus:capture_settings:{uuid}', setting_keys)):
-                    if v is None:
-                        continue
-                    if k in ['url', 'document_name', 'browser', 'device_name', 'user_agent',
-                             'referer', 'timezone_id', 'locale', 'color_scheme']:
-                        # string
-                        to_capture[k] = v.decode()  # type: ignore[literal-required]
-                    elif k in ['cookies', 'http_credentials', 'viewport', 'geolocation']:
-                        # dicts or list
-                        to_capture[k] = json.loads(v)  # type: ignore[literal-required]
-                    elif k in ['proxy', 'headers']:
-                        # can be dict or str
-                        try:
-                            to_capture[k] = json.loads(v)  # type: ignore[literal-required]
-                        except Exception:
-                            to_capture[k] = v.decode()  # type: ignore[literal-required]
-                    elif k in ['general_timeout_in_sec', 'depth']:
-                        # int
-                        to_capture[k] = int(v)  # type: ignore[literal-required]
-                    elif k in ['rendered_hostname_only', 'with_favicon', 'allow_tracking']:
-                        # bool
-                        to_capture[k] = bool(int(v))  # type: ignore[literal-required]
-                    elif k == 'document':
-                        document_as_bytes = b64decode(v)
-                    else:
-                        raise CaptureSettingsError(f'Unexpected setting: {k}: {v}')
-            except CaptureSettingsError as e:
-                raise e
-            except Exception as e:
-                raise CaptureSettingsError(f'Error while preparing settings: {e}')
+            _to_capture = self.redis.hgetall(f'lacus:capture_settings:{uuid}')
+
+            if not _to_capture:
+                result = {'error': f'No capture settings for {uuid}'}
+                raise CaptureError(f'No capture settings for {uuid}')
 
-            if not to_capture:
-                all_entries = self.redis.hgetall(f'lacus:capture_settings:{uuid}')
-                result = {'error': f'No capture settings for {uuid} - {all_entries}'}
-                raise CaptureError(f'No capture settings for {uuid} - {all_entries}')
+            try:
+                to_capture = CaptureSettings(**{k.decode(): v.decode() for k, v in _to_capture.items()})
+            except ValidationError as e:
+                logger.warning(f'Settings invalid: {e}')
+                raise CaptureSettingsError('Invalid settings', e)
 
-            if document_as_bytes:
+            if to_capture.document:
                 # we do not have a URL yet.
-                name = to_capture.pop('document_name', None)
-                if not name:
-                    raise CaptureSettingsError('No document name provided, settings are invalid')
-                if not Path(name).suffix:
-                    # The browser will simply display the file as text if there is no extension.
-                    # Just add HTML as a fallback, as it will be the most comon one.
-                    name = f'{name}.html'
-                document_name = Path(name).name
-                tmp_f = NamedTemporaryFile(suffix=document_name, delete=False)
+                document_as_bytes = b64decode(to_capture.document)
+                tmp_f = NamedTemporaryFile(suffix=to_capture.document_name, delete=False)
                 with open(tmp_f.name, "wb") as f:
                     f.write(document_as_bytes)
                 url = f'file://{tmp_f.name}'
-            elif to_capture.get('url') and to_capture['url'] is not None:
-                url = to_capture['url'].strip()
-                url = refang(url)  # In case we get a defanged url at this stage.
-                if url.lower().startswith('file:') and self.only_global_lookups:
+            elif to_capture.url:
+                if to_capture.url.lower().startswith('file:') and self.only_global_lookups:
                     result = {'error': f'Not allowed to capture a file on disk: {url}'}
                     raise CaptureError(f'Not allowed to capture a file on disk: {url}')
-                if (not url.lower().startswith('data:')
-                        and not url.lower().startswith('http:')
-                        and not url.lower().startswith('https:')
-                        and not url.lower().startswith('file:')):
-                    url = f'http://{url}'
+                url = to_capture.url
             else:
                 result = {'error': f'No valid URL to capture for {uuid} - {to_capture}'}
                 raise CaptureError(f'No valid URL to capture for {uuid} - {to_capture}')
@@ -479,7 +389,7 @@ class LacusCore():
             except Exception as e:
                 result = {'error': f'Invalid URL: {url} - {e}'}
                 raise CaptureError(f'Invalid URL: {url} - {e}')
-            proxy = to_capture.get('proxy')
+            proxy = to_capture.proxy
             if self.tor_proxy:
                 # check if onion or forced
                 if (proxy == 'force_tor'  # if the proxy is set to "force_tor", we use the pre-configured tor proxy, regardless the URL.
@@ -521,9 +431,12 @@ class LacusCore():
                     result = {'error': f'Unable to find hostname or IP in the query: "{url}".'}
                     raise CaptureError(f'Unable to find hostname or IP in the query: "{url}".')
 
+            # Set default as chromium
             browser_engine: BROWSER = "chromium"
-            if to_capture.get('user_agent'):
-                parsed_string = user_agent_parser.ParseUserAgent(to_capture.get('user_agent'))
+            if to_capture.browser:
+                browser_engine = to_capture.browser
+            elif to_capture.user_agent:
+                parsed_string = user_agent_parser.ParseUserAgent(to_capture.user_agent)
                 browser_family = parsed_string['family'].lower()
                 if browser_family.startswith('chrom'):
                     browser_engine = 'chromium'
@@ -533,13 +446,14 @@ class LacusCore():
                     browser_engine = 'webkit'
 
             cookies: list[dict[str, Any]] = []
-            if to_capture.get('cookies') and to_capture['cookies'] is not None:
+            if to_capture.cookies:
                 # In order to properly pass the cookies to playwright,
                 # each of then must have a name, a value and either a domain + path or a URL
                 # Name and value are mandatory, and we cannot auto-fill them.
                 # If the cookie doesn't have a domain + path OR a URL, we fill the domain
                 # with the hostname of the URL we try to capture and the path with "/"
-                for cookie in to_capture['cookies']:
+                # NOTE: these changes can only be done here because we need the URL.
+                for cookie in to_capture.cookies:
                     if len(cookie) == 1:
                         # we have a cookie in the format key: value
                         name, value = cookie.popitem()
@@ -557,21 +471,21 @@ class LacusCore():
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
                 async with Capture(
                         browser=browser_engine,
-                        device_name=to_capture.get('device_name'),
+                        device_name=to_capture.device_name,
                         proxy=proxy,
-                        general_timeout_in_sec=to_capture.get('general_timeout_in_sec'),
+                        general_timeout_in_sec=to_capture.general_timeout_in_sec,
                         loglevel=self.master_logger.getEffectiveLevel(),
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
-                    capture.headers = to_capture.get('headers')  # type: ignore[assignment]
+                    capture.headers = to_capture.headers  # type: ignore[assignment]
                     capture.cookies = cookies  # type: ignore[assignment]
-                    capture.viewport = to_capture.get('viewport')  # type: ignore[assignment]
-                    capture.user_agent = to_capture.get('user_agent')  # type: ignore[assignment]
-                    capture.http_credentials = to_capture.get('http_credentials')  # type: ignore[assignment]
-                    capture.geolocation = to_capture.get('geolocation')  # type: ignore[assignment]
-                    capture.timezone_id = to_capture.get('timezone_id')  # type: ignore[assignment]
-                    capture.locale = to_capture.get('locale')  # type: ignore[assignment]
-                    capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
+                    capture.viewport = to_capture.viewport  # type: ignore[assignment]
+                    capture.user_agent = to_capture.user_agent  # type: ignore[assignment]
+                    capture.http_credentials = to_capture.http_credentials  # type: ignore[assignment]
+                    capture.geolocation = to_capture.geolocation  # type: ignore[assignment]
+                    capture.timezone_id = to_capture.timezone_id  # type: ignore[assignment]
+                    capture.locale = to_capture.locale  # type: ignore[assignment]
+                    capture.color_scheme = to_capture.color_scheme  # type: ignore[assignment]
 
                     # make sure the initialization doesn't take too long
                     init_timeout = max(self.max_capture_time / 10, 5)
@@ -586,11 +500,11 @@ class LacusCore():
                     try:
                         async with timeout(self.max_capture_time) as capture_timeout:
                             playwright_result = await capture.capture_page(
-                                url, referer=to_capture.get('referer'),
-                                depth=to_capture.get('depth', 0),
-                                rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
-                                with_favicon=to_capture.get('with_favicon', False),
-                                allow_tracking=to_capture.get('allow_tracking', False),
+                                url, referer=to_capture.referer,
+                                depth=to_capture.depth,
+                                rendered_hostname_only=to_capture.rendered_hostname_only,
+                                with_favicon=to_capture.with_favicon,
+                                allow_tracking=to_capture.allow_tracking,
                                 max_depth_capture_time=self.max_capture_time)
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
                         timeout_expired(capture_timeout, logger, 'Capture took too long.')
@@ -603,7 +517,7 @@ class LacusCore():
                             stats_pipeline.zincrby(f'stats:{today}:errors', 1, result['error_name'])
             except RetryCapture as e:
                 raise e
-            except PlaywrightCaptureException as e:
+            except (PlaywrightCaptureException, InvalidPlaywrightParameter) as e:
                 logger.warning(f'Invalid parameters for the capture of {url} - {e}')
                 result = {'error': f'Invalid parameters for the capture of {url} - {e}'}
                 raise CaptureError(f'Invalid parameters for the capture of {url} - {e}')
@@ -669,7 +583,7 @@ class LacusCore():
             #       from the lacus:ongoing sorted set (it is definitely not ongoing anymore)
             #       and optionally re-added to lacus:to_capture if re want to retry it
 
-            if to_capture.get('document'):
+            if to_capture.document:
                 os.unlink(tmp_f.name)
 
             if retry:

{lacuscore-1.10.0.dist-info → lacuscore-1.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.10.0
+Version: 1.10.2
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -28,7 +28,8 @@ Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "do
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
 Requires-Dist: dnspython (>=2.6.1,<3.0.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.25.0,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.25.6,<2.0.0)
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.7,<6.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)

lacuscore-1.10.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
+lacuscore/helpers.py,sha256=y7LWHUMMOrB9-qWNoz_2zTP_yazf_opxXykhYjpNves,9640
+lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
+lacuscore/lacuscore.py,sha256=eQMJBf0ZYkCkACu3OWCdidpNEYn0wZ1piIVHP1kv_kU,39372
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=8WbdJdKnGeFCxt9gtCNLI9vAQQZbsy2I5PRQpHP7XFU,1916
+lacuscore-1.10.2.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.10.2.dist-info/METADATA,sha256=xzU2dgvzO_avbhZm4cdy0DWoWR-R9IZwmLocuKBZjQg,2670
+lacuscore-1.10.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lacuscore-1.10.2.dist-info/RECORD,,

lacuscore-1.10.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-lacuscore/__init__.py,sha256=hM4lKoPNybDCUMWdXTVVI1gRk_riLvRZ7IwFbamZLzE,341
-lacuscore/helpers.py,sha256=lULN7HhY-4a4HG-ybIt4jO3wEGTxkm_jKNqsGpNZo4Y,2711
-lacuscore/lacus_monitoring.py,sha256=UOfE_1-_rhVeKJXQ_m9XxYkr7VwyQnA6iK-x_tcXJfo,2775
-lacuscore/lacuscore.py,sha256=n0hjyuJpz5G4s5mRzTAtSCHaPtPqdMaTEO7o-2N88JI,44048
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore/task_logger.py,sha256=8WbdJdKnGeFCxt9gtCNLI9vAQQZbsy2I5PRQpHP7XFU,1916
-lacuscore-1.10.0.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.10.0.dist-info/METADATA,sha256=PmC32J2hEc1hyXiX4wQOjAcnJDLlVIqvj-Yw8pJc13Q,2629
-lacuscore-1.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lacuscore-1.10.0.dist-info/RECORD,,