lacuscore 1.13.0__tar.gz → 1.13.2__tar.gz

{lacuscore-1.13.0 → lacuscore-1.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.13.0
+Version: 1.13.2
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -21,15 +21,16 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: docs
-Requires-Dist: Sphinx (>=8.1.3) ; (python_version >= "3.10") and (extra == "docs")
+Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.28.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.28.1)
 Requires-Dist: pydantic (>=2.10.6)
 Requires-Dist: redis[hiredis] (>=5.2.1)
 Requires-Dist: requests (>=2.32.3)
+Requires-Dist: typing-extensions (>=4.12.2,<5.0.0) ; python_version < "3.12"
 Requires-Dist: ua-parser[regex] (>=1.0.1)
 Project-URL: Documentation, https://lacuscore.readthedocs.io/en/latest/
 Project-URL: Issues, https://github.com/ail-project/issues

{lacuscore-1.13.0 → lacuscore-1.13.2}/lacuscore/helpers.py

@@ -3,10 +3,11 @@
 from __future__ import annotations
 
 import json
+import sys
 
 from enum import IntEnum, unique
 from logging import LoggerAdapter
-from typing import Any, TypedDict, Literal
+from typing import Any, Literal
 from collections.abc import MutableMapping, Mapping
 
 from defang import refang
@@ -15,6 +16,14 @@ from pydantic_core import from_json
 
 from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 
+from playwright._impl._api_structures import Cookie  # , StorageState
+
+
+if sys.version_info >= (3, 12):
+    from typing import TypedDict
+else:
+    from typing_extensions import TypedDict
+
 
 class LacusCoreException(Exception):
     pass
@@ -71,7 +80,10 @@ class CaptureResponseJson(TypedDict, total=False):
     status: int
     last_redirected_url: str | None
     har: dict[str, Any] | None
-    cookies: list[dict[str, str]] | None
+    cookies: list[Cookie] | None
+    # NOTE: should be that, but StorageState doesn't define the indexeddb
+    # storage: StorageState | None
+    storage: dict[str, Any] | None
     error: str | None
     html: str | None
     png: str | None
@@ -93,7 +105,10 @@ class CaptureSettings(BaseModel):
     user_agent: str | None = None
     proxy: str | dict[str, str] | None = None
     general_timeout_in_sec: int | None = None
-    cookies: list[dict[str, Any]] | None = None
+    cookies: list[Cookie] | None = None
+    # NOTE: should be that, but StorageState doesn't define the indexeddb
+    # storage: StorageState | None = None
+    storage: dict[str, Any] | None = None
     headers: dict[str, str] | None = None
     http_credentials: dict[str, str] | None = None
     geolocation: dict[str, float] | None = None
@@ -197,10 +212,51 @@ class CaptureSettings(BaseModel):
         if not cookies:
             return None
         if isinstance(cookies, str):
-            # Cookies are invalid, ignoring.
-            pass
-        elif isinstance(cookies, list):
-            return cookies
+            # might be a json dump, try to load it and ignore otherwise
+            try:
+                cookies = json.loads(cookies)
+            except json.JSONDecodeError as e:
+                print(e)
+                # Cookies are invalid, ignoring.
+                return None
+        if isinstance(cookies, dict):
+            # might be a single cookie in the format name: value, make it a list
+            cookies = [cookies]
+        if isinstance(cookies, list):
+            # make sure the cookies are in the right format
+            to_return = []
+            for cookie in cookies:
+                if isinstance(cookie, dict):
+                    if 'name' in cookie and 'value' in cookie:
+                        to_return.append(cookie)
+                    elif len(cookie) == 1:
+                        # {'name': 'value'} => {'name': 'name', 'value': 'value'}
+                        name, value = cookie.popitem()
+                        if name and value:
+                            to_return.append({'name': name, 'value': value})
+                    else:
+                        # invalid cookie, ignoring
+                        pass
+            return to_return
+        return None
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def load_storage_json(cls, storage: Any) -> dict[str, Any] | None:
+        """That's the storage as exported from Playwright:
+            https://playwright.dev/python/docs/api/class-browsercontext#browser-context-storage-state
+        """
+        if not storage:
+            return None
+        if isinstance(storage, str):
+            # might be a json dump, try to load it and ignore otherwise
+            try:
+                storage = json.loads(storage)
+            except json.JSONDecodeError:
+                # storage is invalid, ignoring.
+                return None
+        if isinstance(storage, dict) and 'cookies' in storage and 'origins' in storage:
+            return storage
         return None
 
     @field_validator('headers', mode='before')

{lacuscore-1.13.0 → lacuscore-1.13.2}/lacuscore/lacuscore.py

@@ -18,7 +18,7 @@ from base64 import b64decode, b64encode
 from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
 from tempfile import NamedTemporaryFile
-from typing import Literal, Any, overload, cast
+from typing import Literal, Any, overload, cast, TYPE_CHECKING
 from collections.abc import Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
@@ -55,6 +55,9 @@ else:
         if timeout_cm.expired():
             logger.warning(f'Timeout expired: {error_message}')
 
+if TYPE_CHECKING:
+    from playwright._impl._api_structures import Cookie
+
 
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
@@ -130,6 +133,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
+                storage: dict[str, Any] | None=None,
                 headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
@@ -161,6 +165,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
+                storage: dict[str, Any] | None=None,
                 headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
@@ -194,6 +199,7 @@ class LacusCore():
         :param proxy: SOCKS5 proxy to use for capturing
         :param general_timeout_in_sec: The capture will raise a timeout it it takes more than that time
         :param cookies: A list of cookies
+        :param storage: A storage state from another capture
         :param headers: The headers to pass to the capture
         :param http_credentials: HTTP Credentials to pass to the capture
         :param geolocation: Geolocation of the browser to pass to the capture
@@ -222,7 +228,7 @@ class LacusCore():
                         'browser': browser, 'device_name': device_name,
                         'user_agent': user_agent, 'proxy': proxy,
                         'general_timeout_in_sec': general_timeout_in_sec,
-                        'cookies': cookies, 'headers': headers,
+                        'cookies': cookies, 'storage': storage, 'headers': headers,
                         'http_credentials': http_credentials, 'geolocation': geolocation,
                         'timezone_id': timezone_id, 'locale': locale,
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
@@ -231,7 +237,6 @@ class LacusCore():
                         # Quietly force it to true if headed is not allowed.
                         'headless': headless if self.headed_allowed else True,
                         'max_retries': max_retries}
-
         try:
             to_enqueue = CaptureSettings(**settings)
         except ValidationError as e:
@@ -244,7 +249,6 @@ class LacusCore():
                 if isinstance(existing_uuid, bytes):
                     return existing_uuid.decode()
                 return existing_uuid
-
         if uuid:
             # Make sure we do not already have a capture with that UUID
             if self.get_capture_status(uuid) == CaptureStatus.UNKNOWN:
@@ -467,7 +471,7 @@ class LacusCore():
                 else:
                     browser_engine = 'webkit'
 
-            cookies: list[dict[str, Any]] = []
+            cookies: list[Cookie] = []
             if to_capture.cookies:
                 # In order to properly pass the cookies to playwright,
                 # each of then must have a name, a value and either a domain + path or a URL
@@ -476,18 +480,16 @@ class LacusCore():
                 # with the hostname of the URL we try to capture and the path with "/"
                 # NOTE: these changes can only be done here because we need the URL.
                 for cookie in to_capture.cookies:
-                    if len(cookie) == 1:
-                        # we have a cookie in the format key: value
-                        name, value = cookie.popitem()
-                        cookie = {'name': name, 'value': value}
                     if 'name' not in cookie or 'value' not in cookie:
                         logger.warning(f'Invalid cookie: {cookie}')
                         continue
                     if 'domain' not in cookie and 'url' not in cookie:
+                        if not splitted_url.hostname:
+                            # If for any reason we cannot get the hostname there, ignore the cookie
+                            continue
                         cookie['domain'] = splitted_url.hostname
                         cookie['path'] = '/'
                     cookies.append(cookie)
-
             try:
                 logger.debug(f'Capturing {url}')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
@@ -502,6 +504,7 @@ class LacusCore():
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers  # type: ignore[assignment]
                     capture.cookies = cookies  # type: ignore[assignment]
+                    capture.storage = to_capture.storage  # type: ignore[assignment]
                     capture.viewport = to_capture.viewport  # type: ignore[assignment]
                     capture.user_agent = to_capture.user_agent  # type: ignore[assignment]
                     capture.http_credentials = to_capture.http_credentials  # type: ignore[assignment]
@@ -665,6 +668,8 @@ class LacusCore():
             hash_to_set['har'] = pickle.dumps(results['har'])
         if results.get('cookies'):
             hash_to_set['cookies'] = pickle.dumps(results['cookies'])
+        if results.get('storage'):
+            hash_to_set['storage'] = pickle.dumps(results['storage'])
         if results.get('potential_favicons'):
             hash_to_set['potential_favicons'] = pickle.dumps(results['potential_favicons'])
         if results.get('html') and results['html'] is not None:
@@ -684,7 +689,7 @@ class LacusCore():
             hash_to_set['children'] = pickle.dumps(children)
 
         for key in results.keys():
-            if key in ['har', 'cookies', 'potential_favicons', 'html', 'children'] or not results.get(key):
+            if key in ['har', 'cookies', 'storage', 'potential_favicons', 'html', 'children'] or not results.get(key):
                 continue
             # these entries can be stored directly
             hash_to_set[key] = results[key]  # type: ignore[literal-required]
@@ -709,6 +714,8 @@ class LacusCore():
                 to_return['har'] = pickle.loads(value)
             elif key == b'cookies':
                 to_return['cookies'] = pickle.loads(value)
+            elif key == b'storage':
+                to_return['storage'] = pickle.loads(value)
             elif key == b'potential_favicons':
                 to_return['potential_favicons'] = pickle.loads(value)
             elif key == b'children':

{lacuscore-1.13.0 → lacuscore-1.13.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lacuscore"
-version = "1.13.0"
+version = "1.13.2"
 description = "Core of Lacus, usable as a module"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -15,14 +15,15 @@ dynamic = [ "classifiers" ]
 
 dependencies = [
     "requests (>=2.32.3)",
-    "playwrightcapture[recaptcha] (>=1.28.0)",
+    "playwrightcapture[recaptcha] (>=1.28.1)",
     "defang (>=0.5.3)",
     "ua-parser[regex] (>=1.0.1)",
     "redis [hiredis] (>=5.2.1)",
     "dnspython (>=2.7.0)",
     "async-timeout (>=5.0.1) ; python_version < \"3.11\"",
     "pydantic (>=2.10.6)",
-    "eval-type-backport (>=0.2.2) ; python_version < \"3.10\""
+    "eval-type-backport (>=0.2.2) ; python_version < \"3.10\"",
+    "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.12\""
 ]
 
 [project.urls]
@@ -43,14 +44,14 @@ classifiers = [
 ]
 
 [project.optional-dependencies]
-docs = ["Sphinx (>=8.1.3) ; python_version >= \"3.10\""]
+docs = ["Sphinx (>=8.2.3) ; python_version >= \"3.11\""]
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.15.0"
 types-redis = {version = "^4.6.0.20241004"}
-types-requests = "^2.32.0.20241016"
+types-requests = "^2.32.0.20250306"
 types-beautifulsoup4 = "^4.12.0.20250204"
-pytest = "^8.3.4"
+pytest = "^8.3.5"
 
 [build-system]
 requires = ["poetry-core>=2.0"]