lacuscore 1.17.3__py3-none-any.whl → 1.17.4__py3-none-any.whl

lacuscore/helpers.py

@@ -88,6 +88,7 @@ class CaptureResponseJson(TypedDict, total=False):
     downloaded_filename: str | None
     downloaded_file: str | None
     children: list[CaptureResponseJson] | None
+    trusted_timestamps: dict[str, str] | None
     runtime: float | None
     potential_favicons: list[str] | None
 
@@ -122,6 +123,7 @@ class CaptureSettings(BaseModel):
     allow_tracking: bool = False
     headless: bool = True
     init_script: str | None = None
+    with_trusted_timestamps: bool = False
     force: bool = False
     recapture_interval: int = 300
     priority: int = 0

lacuscore/lacuscore.py

@@ -31,7 +31,7 @@ from dns.asyncresolver import Resolver
 from dns.exception import DNSException
 from dns.exception import Timeout as DNSTimeout
 
-from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter
+from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter, TrustedTimestampSettings
 from pydantic import ValidationError
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
@@ -115,6 +115,7 @@ class LacusCore():
                  only_global_lookups: bool=True,
                  max_retries: int=3,
                  headed_allowed: bool=False,
+                 tt_settings: TrustedTimestampSettings | None=None,
                  loglevel: str | int='INFO') -> None:
         self.master_logger = logging.getLogger(f'{self.__class__.__name__}')
         self.master_logger.setLevel(loglevel)
@@ -126,6 +127,8 @@ class LacusCore():
         self.tor_proxy = tor_proxy
         self.i2p_proxy = i2p_proxy
 
+        self.tt_settings = tt_settings
+
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
         self.headed_allowed = headed_allowed
@@ -167,6 +170,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_screenshot: bool=True,
                 with_favicon: bool=False,
+                with_trusted_timestamps: bool=False,
                 allow_tracking: bool=False,
                 headless: bool=True,
                 max_retries: int | None=None,
@@ -202,6 +206,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_screenshot: bool=True,
                 with_favicon: bool=False,
+                with_trusted_timestamps: bool=False,
                 allow_tracking: bool=False,
                 headless: bool=True,
                 max_retries: int | None=None,
@@ -239,6 +244,7 @@ class LacusCore():
         :param rendered_hostname_only: If depth > 0: only capture URLs with the same hostname as the rendered page
         :param with_screenshot: If False, PlaywrightCapture won't take a screenshot of the rendered URL
         :param with_favicon: If True, PlaywrightCapture will attempt to get the potential favicons for the rendered URL. It is a dirty trick, see this issue for details: https://github.com/Lookyloo/PlaywrightCapture/issues/45
+        :param with_trusted_timestamps: If True, PlaywrightCapture will trigger calls to a remote timestamp service. For that to work, this class must have been initialized with tt_settings. See RFC3161 for details: https://www.rfc-editor.org/rfc/rfc3161
         :param allow_tracking: If True, PlaywrightCapture will attempt to click through the cookie banners. It is totally dependent on the framework used on the website.
         :param headless: Whether to run the browser in headless mode. WARNING: requires to run in a graphical environment.
         :param max_retries: The maximum anount of retries for this capture
@@ -263,6 +269,7 @@ class LacusCore():
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
                         'viewport': viewport, 'referer': referer,
                         'with_screenshot': with_screenshot, 'with_favicon': with_favicon,
+                        'with_trusted_timestamps': with_trusted_timestamps,
                         'allow_tracking': allow_tracking,
                         # Quietly force it to true if headed is not allowed.
                         'headless': headless if self.headed_allowed else True,
@@ -294,6 +301,10 @@ class LacusCore():
         else:
             perma_uuid = str(uuid4())
 
+        if to_enqueue.with_trusted_timestamps and not self.tt_settings:
+            self.master_logger.warning('Cannot trigger trusted timestamp, the remote timestamper service settings are missing.')
+            to_enqueue.with_trusted_timestamps = False
+
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
         p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump())
@@ -568,6 +579,7 @@ class LacusCore():
                         loglevel=self.master_logger.getEffectiveLevel(),
                         headless=to_capture.headless,
                         init_script=to_capture.init_script,
+                        tt_settings=self.tt_settings,
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers
@@ -601,6 +613,7 @@ class LacusCore():
                                 with_screenshot=to_capture.with_screenshot,
                                 with_favicon=to_capture.with_favicon,
                                 allow_tracking=to_capture.allow_tracking,
+                                with_trusted_timestamps=to_capture.with_trusted_timestamps,
                                 max_depth_capture_time=self.max_capture_time)
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
                         timeout_expired(capture_timeout, logger, 'Capture took too long.')
@@ -757,6 +770,8 @@ class LacusCore():
         if results.get('html') and results['html'] is not None:
             # Need to avoid unicode encore errors, and surrogates are not allowed
             hash_to_set['html'] = results['html'].encode('utf-8', 'surrogateescape')
+        if results.get('trusted_timestamps'):
+            hash_to_set['trusted_timestamps'] = pickle.dumps(results['trusted_timestamps'])
         if 'children' in results and results['children'] is not None:
             padding_length = len(str(len(results['children'])))
             children = set()
@@ -771,11 +786,10 @@ class LacusCore():
             hash_to_set['children'] = pickle.dumps(children)
 
         for key in results.keys():
-            if key in ['har', 'cookies', 'storage', 'potential_favicons', 'html', 'children'] or not results.get(key):
+            if key in ['har', 'cookies', 'storage', 'trusted_timestamps', 'potential_favicons', 'html', 'children'] or not results.get(key):
                 continue
             # these entries can be stored directly
             hash_to_set[key] = results[key]  # type: ignore[literal-required]
-
         if hash_to_set:
             pipeline.hset(root_key, mapping=hash_to_set)  # type: ignore[arg-type]
             # Make sure the key expires
@@ -800,6 +814,8 @@ class LacusCore():
                 to_return['storage'] = pickle.loads(value)
             elif key == b'potential_favicons':
                 to_return['potential_favicons'] = pickle.loads(value)
+            elif key == b'trusted_timestamps':
+                to_return['trusted_timestamps'] = pickle.loads(value)
             elif key == b'children':
                 to_return['children'] = []
                 for child_root_key in sorted(pickle.loads(value)):

{lacuscore-1.17.3.dist-info → lacuscore-1.17.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.17.3
+Version: 1.17.4
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -24,9 +24,10 @@ Provides-Extra: docs
 Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
-Requires-Dist: dnspython (>=2.7.0)
+Requires-Dist: dnspython (<2.8) ; python_version < "3.10"
+Requires-Dist: dnspython (>=2.8.0) ; python_version >= "3.10"
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.32.1)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.32.2)
 Requires-Dist: pydantic (>=2.11.7)
 Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.5)

lacuscore-1.17.4.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
+lacuscore/helpers.py,sha256=E_pgZ6vgtOkb_NCIjgy1c041CSy00arL739hPHVxlS8,14184
+lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
+lacuscore/lacuscore.py,sha256=jrFGBFSYTWRquZsZx7KQAfDc8KTT_nfu7-c6mnrh-pk,48275
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
+lacuscore-1.17.4.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.17.4.dist-info/METADATA,sha256=LCRvwvOmSz65zlMO1iWUjpJ5lGnPVhGm8uRQ1juGBm4,2739
+lacuscore-1.17.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lacuscore-1.17.4.dist-info/RECORD,,

lacuscore-1.17.3.dist-info/RECORD

@@ -1,10 +0,0 @@
-lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
-lacuscore/helpers.py,sha256=dTt-FM7SnwEgIeGTCAvREwa1K224iNb16mmaCwcY_eg,14096
-lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=uz5GJoPZrBTfsWNaCm6Bealc5JIOy7GYe78EqoazDyc,47025
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.17.3.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.17.3.dist-info/METADATA,sha256=TC73WMFDvZYoe6OBM_jZ1bg9qBNOvpa85QGdLrajtlI,2654
-lacuscore-1.17.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lacuscore-1.17.3.dist-info/RECORD,,