lacuscore 1.17.2__py3-none-any.whl → 1.17.4__py3-none-any.whl

lacuscore/helpers.py

@@ -88,6 +88,7 @@ class CaptureResponseJson(TypedDict, total=False):
     downloaded_filename: str | None
     downloaded_file: str | None
     children: list[CaptureResponseJson] | None
+    trusted_timestamps: dict[str, str] | None
     runtime: float | None
     potential_favicons: list[str] | None
 
@@ -122,6 +123,7 @@ class CaptureSettings(BaseModel):
     allow_tracking: bool = False
     headless: bool = True
     init_script: str | None = None
+    with_trusted_timestamps: bool = False
     force: bool = False
     recapture_interval: int = 300
     priority: int = 0

lacuscore/lacuscore.py

@@ -9,6 +9,7 @@ import os
 import pickle
 import random
 import re
+import socket
 import sys
 import time
 import unicodedata
@@ -30,7 +31,7 @@ from dns.asyncresolver import Resolver
 from dns.exception import DNSException
 from dns.exception import Timeout as DNSTimeout
 
-from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter
+from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter, TrustedTimestampSettings
 from pydantic import ValidationError
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
@@ -38,6 +39,7 @@ from redis.exceptions import DataError
 
 from . import task_logger
 from .helpers import (
+    LacusCoreException,
     LacusCoreLogAdapter, CaptureError, RetryCapture, CaptureSettingsError,
     CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings)
 
@@ -80,6 +82,19 @@ def _secure_filename(filename: str) -> str:
     return filename
 
 
+def _check_proxy_port_open(proxy: dict[str, str] | str) -> bool:
+    if isinstance(proxy, dict):
+        to_check = proxy['server']
+    else:
+        to_check = proxy
+    splitted_proxy_url = urlsplit(to_check)
+    if not splitted_proxy_url.hostname or not splitted_proxy_url.port:
+        raise LacusCoreException('Invalid pre-defined proxy (needs hostname and port): {proxy}')
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.settimeout(3)
+        return s.connect_ex((splitted_proxy_url.hostname, splitted_proxy_url.port)) == 0
+
+
 class LacusCore():
     """Capture URLs or web enabled documents using PlaywrightCapture.
 
@@ -95,11 +110,12 @@ class LacusCore():
     def __init__(self, redis_connector: Redis[bytes], /, *,
                  max_capture_time: int=3600,
                  expire_results: int=36000,
-                 tor_proxy: str | None=None,
-                 i2p_proxy: str | None=None,
+                 tor_proxy: dict[str, str] | str | None=None,
+                 i2p_proxy: dict[str, str] | str | None=None,
                  only_global_lookups: bool=True,
                  max_retries: int=3,
                  headed_allowed: bool=False,
+                 tt_settings: TrustedTimestampSettings | None=None,
                  loglevel: str | int='INFO') -> None:
         self.master_logger = logging.getLogger(f'{self.__class__.__name__}')
         self.master_logger.setLevel(loglevel)
@@ -107,8 +123,12 @@ class LacusCore():
         self.redis = redis_connector
         self.max_capture_time = max_capture_time
         self.expire_results = expire_results
+
         self.tor_proxy = tor_proxy
         self.i2p_proxy = i2p_proxy
+
+        self.tt_settings = tt_settings
+
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
         self.headed_allowed = headed_allowed
@@ -150,6 +170,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_screenshot: bool=True,
                 with_favicon: bool=False,
+                with_trusted_timestamps: bool=False,
                 allow_tracking: bool=False,
                 headless: bool=True,
                 max_retries: int | None=None,
@@ -185,6 +206,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_screenshot: bool=True,
                 with_favicon: bool=False,
+                with_trusted_timestamps: bool=False,
                 allow_tracking: bool=False,
                 headless: bool=True,
                 max_retries: int | None=None,
@@ -222,6 +244,7 @@ class LacusCore():
         :param rendered_hostname_only: If depth > 0: only capture URLs with the same hostname as the rendered page
         :param with_screenshot: If False, PlaywrightCapture won't take a screenshot of the rendered URL
         :param with_favicon: If True, PlaywrightCapture will attempt to get the potential favicons for the rendered URL. It is a dirty trick, see this issue for details: https://github.com/Lookyloo/PlaywrightCapture/issues/45
+        :param with_trusted_timestamps: If True, PlaywrightCapture will trigger calls to a remote timestamp service. For that to work, this class must have been initialized with tt_settings. See RFC3161 for details: https://www.rfc-editor.org/rfc/rfc3161
         :param allow_tracking: If True, PlaywrightCapture will attempt to click through the cookie banners. It is totally dependent on the framework used on the website.
         :param headless: Whether to run the browser in headless mode. WARNING: requires to run in a graphical environment.
         :param max_retries: The maximum anount of retries for this capture
@@ -246,6 +269,7 @@ class LacusCore():
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
                         'viewport': viewport, 'referer': referer,
                         'with_screenshot': with_screenshot, 'with_favicon': with_favicon,
+                        'with_trusted_timestamps': with_trusted_timestamps,
                         'allow_tracking': allow_tracking,
                         # Quietly force it to true if headed is not allowed.
                         'headless': headless if self.headed_allowed else True,
@@ -277,6 +301,10 @@ class LacusCore():
         else:
             perma_uuid = str(uuid4())
 
+        if to_enqueue.with_trusted_timestamps and not self.tt_settings:
+            self.master_logger.warning('Cannot trigger trusted timestamp, the remote timestamper service settings are missing.')
+            to_enqueue.with_trusted_timestamps = False
+
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
         p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump())
@@ -453,18 +481,26 @@ class LacusCore():
             proxy = to_capture.proxy
             if self.tor_proxy:
                 # check if onion or forced
-                if (proxy == 'force_tor'  # if the proxy is set to "force_tor", we use the pre-configured tor proxy, regardless the URL.
+                if (proxy == 'force_tor'  # if the proxy is set to "force_tor", we use the pre-configured tor proxy, regardless the URL, legacy feature.
                         or (not proxy  # if the TLD is "onion", we use the pre-configured tor proxy
                             and splitted_url.netloc
                             and splitted_url.hostname
                             and splitted_url.hostname.split('.')[-1] == 'onion')):
+                    if not _check_proxy_port_open(self.tor_proxy):
+                        logger.critical(f'Unable to connect to the default tor proxy: {self.tor_proxy}')
+                        raise CaptureError('The selected tor proxy is unreachable, unable to run the capture.')
                     proxy = self.tor_proxy
-            elif self.i2p_proxy:
+                    logger.info('Using the default tor proxy.')
+            if self.i2p_proxy:
                 if (not proxy  # if the TLD is "i2p", we use the pre-configured I2P proxy
                         and splitted_url.netloc
                         and splitted_url.hostname
                         and splitted_url.hostname.split('.')[-1] == 'i2p'):
+                    if not _check_proxy_port_open(self.i2p_proxy):
+                        logger.critical(f'Unable to connect to the default tor proxy: {self.i2p_proxy}')
+                        raise CaptureError('The selected I2P proxy is unreachable, unable to run the capture.')
                     proxy = self.i2p_proxy
+                    logger.info('Using the default I2P proxy.')
 
             if self.only_global_lookups and not proxy and splitted_url.scheme not in ['data', 'file']:
                 # not relevant if we also have a proxy, or the thing to capture is a data URI or a file on disk
@@ -543,6 +579,7 @@ class LacusCore():
                         loglevel=self.master_logger.getEffectiveLevel(),
                         headless=to_capture.headless,
                         init_script=to_capture.init_script,
+                        tt_settings=self.tt_settings,
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers
@@ -576,6 +613,7 @@ class LacusCore():
                                 with_screenshot=to_capture.with_screenshot,
                                 with_favicon=to_capture.with_favicon,
                                 allow_tracking=to_capture.allow_tracking,
+                                with_trusted_timestamps=to_capture.with_trusted_timestamps,
                                 max_depth_capture_time=self.max_capture_time)
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
                         timeout_expired(capture_timeout, logger, 'Capture took too long.')
@@ -732,6 +770,8 @@ class LacusCore():
         if results.get('html') and results['html'] is not None:
             # Need to avoid unicode encore errors, and surrogates are not allowed
             hash_to_set['html'] = results['html'].encode('utf-8', 'surrogateescape')
+        if results.get('trusted_timestamps'):
+            hash_to_set['trusted_timestamps'] = pickle.dumps(results['trusted_timestamps'])
         if 'children' in results and results['children'] is not None:
             padding_length = len(str(len(results['children'])))
             children = set()
@@ -746,11 +786,10 @@ class LacusCore():
             hash_to_set['children'] = pickle.dumps(children)
 
         for key in results.keys():
-            if key in ['har', 'cookies', 'storage', 'potential_favicons', 'html', 'children'] or not results.get(key):
+            if key in ['har', 'cookies', 'storage', 'trusted_timestamps', 'potential_favicons', 'html', 'children'] or not results.get(key):
                 continue
             # these entries can be stored directly
             hash_to_set[key] = results[key]  # type: ignore[literal-required]
-
         if hash_to_set:
             pipeline.hset(root_key, mapping=hash_to_set)  # type: ignore[arg-type]
             # Make sure the key expires
@@ -775,6 +814,8 @@ class LacusCore():
                 to_return['storage'] = pickle.loads(value)
             elif key == b'potential_favicons':
                 to_return['potential_favicons'] = pickle.loads(value)
+            elif key == b'trusted_timestamps':
+                to_return['trusted_timestamps'] = pickle.loads(value)
             elif key == b'children':
                 to_return['children'] = []
                 for child_root_key in sorted(pickle.loads(value)):

{lacuscore-1.17.2.dist-info → lacuscore-1.17.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.17.2
+Version: 1.17.4
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -24,9 +24,10 @@ Provides-Extra: docs
 Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
 Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
-Requires-Dist: dnspython (>=2.7.0)
+Requires-Dist: dnspython (<2.8) ; python_version < "3.10"
+Requires-Dist: dnspython (>=2.8.0) ; python_version >= "3.10"
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.32.1)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.32.2)
 Requires-Dist: pydantic (>=2.11.7)
 Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.5)

lacuscore-1.17.4.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
+lacuscore/helpers.py,sha256=E_pgZ6vgtOkb_NCIjgy1c041CSy00arL739hPHVxlS8,14184
+lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
+lacuscore/lacuscore.py,sha256=jrFGBFSYTWRquZsZx7KQAfDc8KTT_nfu7-c6mnrh-pk,48275
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
+lacuscore-1.17.4.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.17.4.dist-info/METADATA,sha256=LCRvwvOmSz65zlMO1iWUjpJ5lGnPVhGm8uRQ1juGBm4,2739
+lacuscore-1.17.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lacuscore-1.17.4.dist-info/RECORD,,

lacuscore-1.17.2.dist-info/RECORD

@@ -1,10 +0,0 @@
-lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
-lacuscore/helpers.py,sha256=dTt-FM7SnwEgIeGTCAvREwa1K224iNb16mmaCwcY_eg,14096
-lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=fU8rojUgdxHlLjMwqDcZetwMBPVEVAS341g4xcusK4g,45682
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.17.2.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.17.2.dist-info/METADATA,sha256=t83EragMDEZzYsfX53l1bGbiiItE8hUTVM6nm7ESc70,2654
-lacuscore-1.17.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lacuscore-1.17.2.dist-info/RECORD,,