lacuscore 1.12.9__tar.gz → 1.13.0__tar.gz

{lacuscore-1.12.9 → lacuscore-1.13.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.12.9
+Version: 1.13.0
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.9.2,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Information Technology
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -27,7 +26,7 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.27.7)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.28.0)
 Requires-Dist: pydantic (>=2.10.6)
 Requires-Dist: redis[hiredis] (>=5.2.1)
 Requires-Dist: requests (>=2.32.3)

{lacuscore-1.12.9 → lacuscore-1.13.0}/lacuscore/helpers.py

@@ -105,6 +105,7 @@ class CaptureSettings(BaseModel):
     referer: str | None = None
     with_favicon: bool = True
     allow_tracking: bool = False
+    headless: bool = True
     force: bool = False
     recapture_interval: int = 300
     priority: int = 0

{lacuscore-1.12.9 → lacuscore-1.13.0}/lacuscore/lacuscore.py

@@ -12,7 +12,6 @@ import re
 import sys
 import time
 import unicodedata
-import zlib
 
 from asyncio import Task
 from base64 import b64decode, b64encode
@@ -95,15 +94,19 @@ class LacusCore():
                  tor_proxy: str | None=None,
                  only_global_lookups: bool=True,
                  max_retries: int=3,
+                 headed_allowed: bool=False,
                  loglevel: str | int='INFO') -> None:
         self.master_logger = logging.getLogger(f'{self.__class__.__name__}')
         self.master_logger.setLevel(loglevel)
+
         self.redis = redis_connector
         self.max_capture_time = max_capture_time
         self.expire_results = expire_results
         self.tor_proxy = tor_proxy
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
+        self.headed_allowed = headed_allowed
+
         self.dnsresolver: Resolver = Resolver()
         self.dnsresolver.cache = Cache(900)
         self.dnsresolver.timeout = 2
@@ -139,6 +142,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                headless: bool=True,
                 max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
@@ -169,6 +173,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                headless: bool=True,
                 max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
@@ -201,6 +206,7 @@ class LacusCore():
         :param rendered_hostname_only: If depth > 0: only capture URLs with the same hostname as the rendered page
         :param with_favicon: If True, PlaywrightCapture will attempt to get the potential favicons for the rendered URL. It is a dirty trick, see this issue for details: https://github.com/Lookyloo/PlaywrightCapture/issues/45
         :param allow_tracking: If True, PlaywrightCapture will attempt to click through the cookie banners. It is totally dependent on the framework used on the website.
+        :param headless: Whether to run the browser in headless mode. WARNING: requires to run in a graphical environment.
         :param max_retries: The maximum anount of retries for this capture
 
         :param force: Force recapture, even if the same one was already done within the recapture_interval
@@ -221,7 +227,10 @@ class LacusCore():
                         'timezone_id': timezone_id, 'locale': locale,
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
                         'viewport': viewport, 'referer': referer, 'with_favicon': with_favicon,
-                        'allow_tracking': allow_tracking, 'max_retries': max_retries}
+                        'allow_tracking': allow_tracking,
+                        # Quietly force it to true if headed is not allowed.
+                        'headless': headless if self.headed_allowed else True,
+                        'max_retries': max_retries}
 
         try:
             to_enqueue = CaptureSettings(**settings)
@@ -488,6 +497,7 @@ class LacusCore():
                         proxy=proxy,
                         general_timeout_in_sec=to_capture.general_timeout_in_sec,
                         loglevel=self.master_logger.getEffectiveLevel(),
+                        headless=to_capture.headless,
                         uuid=uuid) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
                     capture.headers = to_capture.headers  # type: ignore[assignment]
@@ -691,12 +701,6 @@ class LacusCore():
         if root_key is None:
             root_key = f'lacus:capture_results_hash:{capture_uuid}'
 
-            if not self.redis.exists(root_key):
-                if old_response := self.redis.get(f'lacus:capture_results:{capture_uuid}'):
-                    # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
-                    return pickle.loads(zlib.decompress(old_response))
-                return None
-
         # New format and capture done
 
         to_return: CaptureResponse = {}

{lacuscore-1.12.9 → lacuscore-1.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lacuscore"
-version = "1.12.9"
+version = "1.13.0"
 description = "Core of Lacus, usable as a module"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -9,13 +9,13 @@ license = "BSD-3-Clause"
 repository = "https://github.com/ail-project/LacusCore"
 documentation = "https://lacuscore.readthedocs.io/en/latest/"
 readme = "README.md"
-requires-python = ">=3.9,<4.0"
+requires-python = ">=3.9.2,<4.0"
 
 dynamic = [ "classifiers" ]
 
 dependencies = [
     "requests (>=2.32.3)",
-    "playwrightcapture[recaptcha] (>=1.27.7)",
+    "playwrightcapture[recaptcha] (>=1.28.0)",
     "defang (>=0.5.3)",
     "ua-parser[regex] (>=1.0.1)",
     "redis [hiredis] (>=5.2.1)",
@@ -46,10 +46,10 @@ classifiers = [
 docs = ["Sphinx (>=8.1.3) ; python_version >= \"3.10\""]
 
 [tool.poetry.group.dev.dependencies]
-mypy = "^1.14.1"
+mypy = "^1.15.0"
 types-redis = {version = "^4.6.0.20241004"}
 types-requests = "^2.32.0.20241016"
-types-beautifulsoup4 = "^4.12.0.20241020"
+types-beautifulsoup4 = "^4.12.0.20250204"
 pytest = "^8.3.4"
 
 [build-system]