lacuscore 1.12.9__py3-none-any.whl → 1.13.0__py3-none-any.whl

lacuscore/helpers.py

@@ -105,6 +105,7 @@ class CaptureSettings(BaseModel):
     referer: str | None = None
     with_favicon: bool = True
     allow_tracking: bool = False
+    headless: bool = True
     force: bool = False
     recapture_interval: int = 300
     priority: int = 0

lacuscore/lacuscore.py

@@ -12,7 +12,6 @@ import re
 import sys
 import time
 import unicodedata
-import zlib
 
 from asyncio import Task
 from base64 import b64decode, b64encode
@@ -95,15 +94,19 @@ class LacusCore():
                  tor_proxy: str | None=None,
                  only_global_lookups: bool=True,
                  max_retries: int=3,
+                 headed_allowed: bool=False,
                  loglevel: str | int='INFO') -> None:
         self.master_logger = logging.getLogger(f'{self.__class__.__name__}')
         self.master_logger.setLevel(loglevel)
+
         self.redis = redis_connector
         self.max_capture_time = max_capture_time
         self.expire_results = expire_results
         self.tor_proxy = tor_proxy
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
+        self.headed_allowed = headed_allowed
+
         self.dnsresolver: Resolver = Resolver()
         self.dnsresolver.cache = Cache(900)
         self.dnsresolver.timeout = 2
@@ -139,6 +142,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                headless: bool=True,
                 max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
@@ -169,6 +173,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                headless: bool=True,
                 max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
@@ -201,6 +206,7 @@ class LacusCore():
         :param rendered_hostname_only: If depth > 0: only capture URLs with the same hostname as the rendered page
         :param with_favicon: If True, PlaywrightCapture will attempt to get the potential favicons for the rendered URL. It is a dirty trick, see this issue for details: https://github.com/Lookyloo/PlaywrightCapture/issues/45
         :param allow_tracking: If True, PlaywrightCapture will attempt to click through the cookie banners. It is totally dependent on the framework used on the website.
+        :param headless: Whether to run the browser in headless mode. WARNING: requires to run in a graphical environment.
         :param max_retries: The maximum anount of retries for this capture
 
         :param force: Force recapture, even if the same one was already done within the recapture_interval
@@ -221,7 +227,10 @@ class LacusCore():
                         'timezone_id': timezone_id, 'locale': locale,
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
                         'viewport': viewport, 'referer': referer, 'with_favicon': with_favicon,
-                        'allow_tracking': allow_tracking, 'max_retries': max_retries}
+                        'allow_tracking': allow_tracking,
+                        # Quietly force it to true if headed is not allowed.
+                        'headless': headless if self.headed_allowed else True,
+                        'max_retries': max_retries}
 
         try:
             to_enqueue = CaptureSettings(**settings)
@@ -488,6 +497,7 @@ class LacusCore():
                         proxy=proxy,
                         general_timeout_in_sec=to_capture.general_timeout_in_sec,
                         loglevel=self.master_logger.getEffectiveLevel(),
+                        headless=to_capture.headless,
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers  # type: ignore[assignment]
@@ -691,12 +701,6 @@ class LacusCore():
         if root_key is None:
             root_key = f'lacus:capture_results_hash:{capture_uuid}'
 
-            if not self.redis.exists(root_key):
-                if old_response := self.redis.get(f'lacus:capture_results:{capture_uuid}'):
-                    # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
-                    return pickle.loads(zlib.decompress(old_response))
-                return None
-
         # New format and capture done
 
         to_return: CaptureResponse = {}

{lacuscore-1.12.9.dist-info → lacuscore-1.13.0.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.12.9
+Version: 1.13.0
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.9.2,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Information Technology
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -27,7 +26,7 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.27.7)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.28.0)
 Requires-Dist: pydantic (>=2.10.6)
 Requires-Dist: redis[hiredis] (>=5.2.1)
 Requires-Dist: requests (>=2.32.3)

lacuscore-1.13.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
+lacuscore/helpers.py,sha256=GKgy8-kvGJLrOv431AtQRTtMSJ5GNtnwsj-K6WqF0EA,10993
+lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
+lacuscore/lacuscore.py,sha256=4yUGTdURX61Os3pvpSqtXP6u9FEhcIb4utaiSUMGIpo,41687
+lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
+lacuscore-1.13.0.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.13.0.dist-info/METADATA,sha256=6X46qhNEVj4QflVHAjAp2U6AI97JdDIOuwaPISTtQfg,2570
+lacuscore-1.13.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+lacuscore-1.13.0.dist-info/RECORD,,

{lacuscore-1.12.9.dist-info → lacuscore-1.13.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any

lacuscore-1.12.9.dist-info/RECORD

@@ -1,10 +0,0 @@
-lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
-lacuscore/helpers.py,sha256=P-OESi6msSqewxp0CqpyJu_fH1lT6rre4FZUK45tUGY,10967
-lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=2MQgfLEi6yUuq-XQeo3XXp5_OaHd4z40g05fKRXR6Qc,41526
-lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.12.9.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.12.9.dist-info/METADATA,sha256=WWk4cu-rAAU1vfCs-9wV2yF23p0UGTF14KTd4iJ9aTU,2618
-lacuscore-1.12.9.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-lacuscore-1.12.9.dist-info/RECORD,,