lacuscore 1.12.10__py3-none-any.whl → 1.13.1__py3-none-any.whl

lacuscore/helpers.py

@@ -15,6 +15,8 @@ from pydantic_core import from_json
 
 from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 
+from playwright._impl._api_structures import Cookie  # , StorageState
+
 
 class LacusCoreException(Exception):
     pass
@@ -71,7 +73,10 @@ class CaptureResponseJson(TypedDict, total=False):
     status: int
     last_redirected_url: str | None
     har: dict[str, Any] | None
-    cookies: list[dict[str, str]] | None
+    cookies: list[Cookie] | None
+    # NOTE: should be that, but StorageState doesn't define the indexeddb
+    # storage: StorageState | None
+    storage: dict[str, Any] | None
     error: str | None
     html: str | None
     png: str | None
@@ -93,7 +98,10 @@ class CaptureSettings(BaseModel):
     user_agent: str | None = None
     proxy: str | dict[str, str] | None = None
     general_timeout_in_sec: int | None = None
-    cookies: list[dict[str, Any]] | None = None
+    cookies: list[Cookie] | None = None
+    # NOTE: should be that, but StorageState doesn't define the indexeddb
+    # storage: StorageState | None = None
+    storage: dict[str, Any] | None = None
     headers: dict[str, str] | None = None
     http_credentials: dict[str, str] | None = None
     geolocation: dict[str, float] | None = None
@@ -197,10 +205,51 @@ class CaptureSettings(BaseModel):
         if not cookies:
             return None
         if isinstance(cookies, str):
-            # Cookies are invalid, ignoring.
-            pass
-        elif isinstance(cookies, list):
-            return cookies
+            # might be a json dump, try to load it and ignore otherwise
+            try:
+                cookies = json.loads(cookies)
+            except json.JSONDecodeError as e:
+                print(e)
+                # Cookies are invalid, ignoring.
+                return None
+        if isinstance(cookies, dict):
+            # might be a single cookie in the format name: value, make it a list
+            cookies = [cookies]
+        if isinstance(cookies, list):
+            # make sure the cookies are in the right format
+            to_return = []
+            for cookie in cookies:
+                if isinstance(cookie, dict):
+                    if 'name' in cookie and 'value' in cookie:
+                        to_return.append(cookie)
+                    elif len(cookie) == 1:
+                        # {'name': 'value'} => {'name': 'name', 'value': 'value'}
+                        name, value = cookie.popitem()
+                        if name and value:
+                            to_return.append({'name': name, 'value': value})
+                    else:
+                        # invalid cookie, ignoring
+                        pass
+            return to_return
+        return None
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def load_storage_json(cls, storage: Any) -> dict[str, Any] | None:
+        """That's the storage as exported from Playwright:
+            https://playwright.dev/python/docs/api/class-browsercontext#browser-context-storage-state
+        """
+        if not storage:
+            return None
+        if isinstance(storage, str):
+            # might be a json dump, try to load it and ignore otherwise
+            try:
+                storage = json.loads(storage)
+            except json.JSONDecodeError:
+                # storage is invalid, ignoring.
+                return None
+        if isinstance(storage, dict) and 'cookies' in storage and 'origins' in storage:
+            return storage
         return None
 
     @field_validator('headers', mode='before')

lacuscore/lacuscore.py

@@ -12,14 +12,13 @@ import re
 import sys
 import time
 import unicodedata
-import zlib
 
 from asyncio import Task
 from base64 import b64decode, b64encode
 from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
 from tempfile import NamedTemporaryFile
-from typing import Literal, Any, overload, cast
+from typing import Literal, Any, overload, cast, TYPE_CHECKING
 from collections.abc import Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
@@ -56,6 +55,9 @@ else:
         if timeout_cm.expired():
             logger.warning(f'Timeout expired: {error_message}')
 
+if TYPE_CHECKING:
+    from playwright._impl._api_structures import Cookie
+
 
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
@@ -131,6 +133,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
+                storage: dict[str, Any] | None=None,
                 headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
@@ -162,6 +165,7 @@ class LacusCore():
                 proxy: str | dict[str, str] | None=None,
                 general_timeout_in_sec: int | None=None,
                 cookies: list[dict[str, Any]] | None=None,
+                storage: dict[str, Any] | None=None,
                 headers: dict[str, str] | None=None,
                 http_credentials: dict[str, str] | None=None,
                 geolocation: dict[str, float] | None=None,
@@ -195,6 +199,7 @@ class LacusCore():
         :param proxy: SOCKS5 proxy to use for capturing
         :param general_timeout_in_sec: The capture will raise a timeout it it takes more than that time
         :param cookies: A list of cookies
+        :param storage: A storage state from another capture
         :param headers: The headers to pass to the capture
         :param http_credentials: HTTP Credentials to pass to the capture
         :param geolocation: Geolocation of the browser to pass to the capture
@@ -223,7 +228,7 @@ class LacusCore():
                         'browser': browser, 'device_name': device_name,
                         'user_agent': user_agent, 'proxy': proxy,
                         'general_timeout_in_sec': general_timeout_in_sec,
-                        'cookies': cookies, 'headers': headers,
+                        'cookies': cookies, 'storage': storage, 'headers': headers,
                         'http_credentials': http_credentials, 'geolocation': geolocation,
                         'timezone_id': timezone_id, 'locale': locale,
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
@@ -232,7 +237,6 @@ class LacusCore():
                         # Quietly force it to true if headed is not allowed.
                         'headless': headless if self.headed_allowed else True,
                         'max_retries': max_retries}
-
         try:
             to_enqueue = CaptureSettings(**settings)
         except ValidationError as e:
@@ -245,7 +249,6 @@ class LacusCore():
                 if isinstance(existing_uuid, bytes):
                     return existing_uuid.decode()
                 return existing_uuid
-
         if uuid:
             # Make sure we do not already have a capture with that UUID
             if self.get_capture_status(uuid) == CaptureStatus.UNKNOWN:
@@ -468,7 +471,7 @@ class LacusCore():
                 else:
                     browser_engine = 'webkit'
 
-            cookies: list[dict[str, Any]] = []
+            cookies: list[Cookie] = []
             if to_capture.cookies:
                 # In order to properly pass the cookies to playwright,
                 # each of then must have a name, a value and either a domain + path or a URL
@@ -477,18 +480,16 @@ class LacusCore():
                 # with the hostname of the URL we try to capture and the path with "/"
                 # NOTE: these changes can only be done here because we need the URL.
                 for cookie in to_capture.cookies:
-                    if len(cookie) == 1:
-                        # we have a cookie in the format key: value
-                        name, value = cookie.popitem()
-                        cookie = {'name': name, 'value': value}
                     if 'name' not in cookie or 'value' not in cookie:
                         logger.warning(f'Invalid cookie: {cookie}')
                         continue
                     if 'domain' not in cookie and 'url' not in cookie:
+                        if not splitted_url.hostname:
+                            # If for any reason we cannot get the hostname there, ignore the cookie
+                            continue
                         cookie['domain'] = splitted_url.hostname
                         cookie['path'] = '/'
                     cookies.append(cookie)
-
             try:
                 logger.debug(f'Capturing {url}')
                 stats_pipeline.sadd(f'stats:{today}:captures', url)
@@ -503,6 +504,7 @@ class LacusCore():
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers  # type: ignore[assignment]
                     capture.cookies = cookies  # type: ignore[assignment]
+                    capture.storage = to_capture.storage  # type: ignore[assignment]
                     capture.viewport = to_capture.viewport  # type: ignore[assignment]
                     capture.user_agent = to_capture.user_agent  # type: ignore[assignment]
                     capture.http_credentials = to_capture.http_credentials  # type: ignore[assignment]
@@ -666,6 +668,8 @@ class LacusCore():
             hash_to_set['har'] = pickle.dumps(results['har'])
         if results.get('cookies'):
             hash_to_set['cookies'] = pickle.dumps(results['cookies'])
+        if results.get('storage'):
+            hash_to_set['storage'] = pickle.dumps(results['storage'])
         if results.get('potential_favicons'):
             hash_to_set['potential_favicons'] = pickle.dumps(results['potential_favicons'])
         if results.get('html') and results['html'] is not None:
@@ -685,7 +689,7 @@ class LacusCore():
             hash_to_set['children'] = pickle.dumps(children)
 
         for key in results.keys():
-            if key in ['har', 'cookies', 'potential_favicons', 'html', 'children'] or not results.get(key):
+            if key in ['har', 'cookies', 'storage', 'potential_favicons', 'html', 'children'] or not results.get(key):
                 continue
             # these entries can be stored directly
             hash_to_set[key] = results[key]  # type: ignore[literal-required]
@@ -702,12 +706,6 @@ class LacusCore():
         if root_key is None:
             root_key = f'lacus:capture_results_hash:{capture_uuid}'
 
-            if not self.redis.exists(root_key):
-                if old_response := self.redis.get(f'lacus:capture_results:{capture_uuid}'):
-                    # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
-                    return pickle.loads(zlib.decompress(old_response))
-                return None
-
         # New format and capture done
 
         to_return: CaptureResponse = {}
@@ -716,6 +714,8 @@ class LacusCore():
                 to_return['har'] = pickle.loads(value)
             elif key == b'cookies':
                 to_return['cookies'] = pickle.loads(value)
+            elif key == b'storage':
+                to_return['storage'] = pickle.loads(value)
             elif key == b'potential_favicons':
                 to_return['potential_favicons'] = pickle.loads(value)
             elif key == b'children':

{lacuscore-1.12.10.dist-info → lacuscore-1.13.1.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.12.10
+Version: 1.13.1
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.9.2,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Information Technology
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -22,12 +21,12 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: docs
-Requires-Dist: Sphinx (>=8.1.3) ; (python_version >= "3.10") and (extra == "docs")
+Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.27.8)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.28.1)
 Requires-Dist: pydantic (>=2.10.6)
 Requires-Dist: redis[hiredis] (>=5.2.1)
 Requires-Dist: requests (>=2.32.3)

lacuscore-1.13.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
+lacuscore/helpers.py,sha256=Nt3oMMDGgl3rDkDujuAaxWtb3cXeSGk2pdXg5lNEqhI,13188
+lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
+lacuscore/lacuscore.py,sha256=fzW-04_pFK_Bqv-66-ei4OLS3nxpHmNEXCn4_2pXhTU,42212
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
+lacuscore-1.13.1.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.13.1.dist-info/METADATA,sha256=IyAbfTlzx6MfiGGSm4xcQsz_l7zlAS4iKdhpc1xlOuA,2570
+lacuscore-1.13.1.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+lacuscore-1.13.1.dist-info/RECORD,,

{lacuscore-1.12.10.dist-info → lacuscore-1.13.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any

lacuscore-1.12.10.dist-info/RECORD

@@ -1,10 +0,0 @@
-lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
-lacuscore/helpers.py,sha256=GKgy8-kvGJLrOv431AtQRTtMSJ5GNtnwsj-K6WqF0EA,10993
-lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=S-qxQJR4WmJHDSAay9tBCWHz2f4uM6z1Elmox7hln54,42049
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.12.10.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.12.10.dist-info/METADATA,sha256=Z2TqBsZDn5K0omvbs0Gv_4TvZBkEAH1RxVO7XfXznl4,2619
-lacuscore-1.12.10.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-lacuscore-1.12.10.dist-info/RECORD,,