lacuscore 1.11.3__tar.gz → 1.12.1__tar.gz

{lacuscore-1.11.3 → lacuscore-1.12.1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.11.3
+Version: 1.12.1
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Information Technology
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -24,15 +23,13 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: docs
-Requires-Dist: Sphinx (<7.2) ; (python_version < "3.9") and (extra == "docs")
 Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9" and python_version < "3.10") and (extra == "docs")
 Requires-Dist: Sphinx (>=8,<9) ; (python_version >= "3.10") and (extra == "docs")
 Requires-Dist: async-timeout (>=4.0.3,<5.0.0) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
-Requires-Dist: dnspython (<2.7) ; python_version < "3.9"
-Requires-Dist: dnspython (>=2.7,<3.0) ; python_version >= "3.9"
+Requires-Dist: dnspython (>=2.7.0,<3.0.0) ; python_version >= "3.9"
 Requires-Dist: eval-type-backport (>=0.2.0,<0.3.0) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.26.3,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.27.0,<2.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: redis[hiredis] (>=5.2.0,<6.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)

{lacuscore-1.11.3 → lacuscore-1.12.1}/lacuscore/helpers.py

@@ -6,7 +6,8 @@ import json
 
 from enum import IntEnum, unique
 from logging import LoggerAdapter
-from typing import MutableMapping, Any, TypedDict, Mapping, Literal
+from typing import Any, TypedDict, Literal
+from collections.abc import MutableMapping, Mapping
 
 from defang import refang  # type: ignore[import-untyped]
 from pydantic import BaseModel, field_validator, model_validator, ValidationError
@@ -107,6 +108,7 @@ class CaptureSettings(BaseModel):
     force: bool = False
     recapture_interval: int = 300
     priority: int = 0
+    max_retries: int | None = None
     uuid: str | None = None
 
     depth: int = 0

{lacuscore-1.11.3 → lacuscore-1.12.1}/lacuscore/lacuscore.py

@@ -19,7 +19,8 @@ from base64 import b64decode, b64encode
 from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
 from tempfile import NamedTemporaryFile
-from typing import Literal, Any, overload, cast, Iterator
+from typing import Literal, Any, overload, cast
+from collections.abc import Iterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
@@ -80,7 +81,7 @@ class LacusCore():
     """Capture URLs or web enabled documents using PlaywrightCapture.
 
     :param redis_connector: Pre-configured connector to a redis instance.
-    :param max_capture time: If the capture takes more than that time, break (in seconds)
+    :param max_capture_time: If the capture takes more than that time, break (in seconds)
     :param tor_proxy: URL to a SOCKS 5 tor proxy. If you have tor installed, this is the default: socks5://127.0.0.1:9050.
     :param only_global_lookups: Discard captures that point to non-public IPs.
     :param max_retries: How many times should we re-try a capture if it failed.
@@ -137,6 +138,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
                 priority: int=0,
@@ -166,6 +168,7 @@ class LacusCore():
                 rendered_hostname_only: bool=True,
                 with_favicon: bool=False,
                 allow_tracking: bool=False,
+                max_retries: int | None=None,
                 force: bool=False,
                 recapture_interval: int=300,
                 priority: int=0,
@@ -197,6 +200,8 @@ class LacusCore():
         :param rendered_hostname_only: If depth > 0: only capture URLs with the same hostname as the rendered page
         :param with_favicon: If True, PlaywrightCapture will attempt to get the potential favicons for the rendered URL. It is a dirty trick, see this issue for details: https://github.com/Lookyloo/PlaywrightCapture/issues/45
         :param allow_tracking: If True, PlaywrightCapture will attempt to click through the cookie banners. It is totally dependent on the framework used on the website.
+        :param max_retries: The maximum anount of retries for this capture
+
         :param force: Force recapture, even if the same one was already done within the recapture_interval
         :param recapture_interval: The time the enqueued settings are kept in memory to avoid duplicates
         :param priority: The priority of the capture
@@ -215,7 +220,7 @@ class LacusCore():
                         'timezone_id': timezone_id, 'locale': locale,
                         'color_scheme': color_scheme, 'java_script_enabled': java_script_enabled,
                         'viewport': viewport, 'referer': referer, 'with_favicon': with_favicon,
-                        'allow_tracking': allow_tracking}
+                        'allow_tracking': allow_tracking, 'max_retries': max_retries}
 
         try:
             to_enqueue = CaptureSettings(**settings)
@@ -371,6 +376,10 @@ class LacusCore():
                 logger.warning(f'Settings invalid: {e}')
                 raise CaptureSettingsError('Invalid settings', e)
 
+            # If the class is initialized with max_retries below the one provided in the settings, we use the lowest value
+            # NOTE: make sure the variable is initialized *before* we raise any RetryCapture
+            max_retries = min([to_capture.max_retries, self.max_retries]) if to_capture.max_retries is not None else self.max_retries
+
             if to_capture.document:
                 # we do not have a URL yet.
                 document_as_bytes = b64decode(to_capture.document)
@@ -544,25 +553,29 @@ class LacusCore():
                 # this is a retry that worked
                 stats_pipeline.sadd(f'stats:{today}:retry_success', url)
         except RetryCapture:
-            # Check if we already re-tried this capture
-            _current_retry = self.redis.get(f'lacus:capture_retry:{uuid}')
-            if _current_retry is None:
-                # No retry yet
-                logger.debug(f'Retrying {url} for the first time.')
-                retry = True
-                self.redis.setex(f'lacus:capture_retry:{uuid}',
-                                 self.max_capture_time * (self.max_retries + 10),
-                                 self.max_retries)
+            if max_retries == 0:
+                error_msg = result['error'] if result.get('error') else 'Unknown error'
+                logger.info(f'Retries disabled for {url}: {error_msg}')
             else:
-                current_retry = int(_current_retry.decode())
-                if current_retry > 0:
-                    logger.debug(f'Retrying {url} for the {self.max_retries - current_retry + 1}th time.')
-                    self.redis.decr(f'lacus:capture_retry:{uuid}')
+                # Check if we already re-tried this capture
+                _current_retry = self.redis.get(f'lacus:capture_retry:{uuid}')
+                if _current_retry is None:
+                    # No retry yet
+                    logger.debug(f'Retrying {url} for the first time.')
                     retry = True
+                    self.redis.setex(f'lacus:capture_retry:{uuid}',
+                                     self.max_capture_time * (max_retries + 10),
+                                     max_retries - 1)
                 else:
-                    error_msg = result['error'] if result.get('error') else 'Unknown error'
-                    logger.info(f'Retried too many times {url}: {error_msg}')
-                    stats_pipeline.sadd(f'stats:{today}:retry_failed', url)
+                    current_retry = int(_current_retry.decode())
+                    if current_retry > 0:
+                        logger.debug(f'Retrying {url} for the {max_retries - current_retry + 1} time.')
+                        self.redis.decr(f'lacus:capture_retry:{uuid}')
+                        retry = True
+                    else:
+                        error_msg = result['error'] if result.get('error') else 'Unknown error'
+                        logger.info(f'Retried too many times {url}: {error_msg}')
+                        stats_pipeline.sadd(f'stats:{today}:retry_failed', url)
         except CaptureError:
             if not result:
                 result = {'error': "No result key, shouldn't happen"}

{lacuscore-1.11.3 → lacuscore-1.12.1}/lacuscore/task_logger.py

@@ -2,7 +2,8 @@
 
 from __future__ import annotations
 
-from typing import Any, Coroutine, Optional, TypeVar, Tuple
+from typing import Any, TypeVar
+from collections.abc import Coroutine
 
 import asyncio
 import functools
@@ -18,12 +19,12 @@ def create_task(
     coroutine: Coroutine[Any, Any, T],
     *,
     name: str,
-    logger: 'LacusCoreLogAdapter',
+    logger: LacusCoreLogAdapter,
     message: str,
-    message_args: Tuple[Any, ...] = (),
-    loop: Optional[asyncio.AbstractEventLoop] = None,
+    message_args: tuple[Any, ...] = (),
+    loop: asyncio.AbstractEventLoop | None = None,
 
-) -> 'asyncio.Task[T]':  # This type annotation has to be quoted for Python < 3.9, see https://www.python.org/dev/peps/pep-0585/
+) -> asyncio.Task[T]:  # This type annotation has to be quoted for Python < 3.9, see https://www.python.org/dev/peps/pep-0585/
     '''
     This helper function wraps a ``loop.create_task(coroutine())`` call and ensures there is
     an exception handler added to the resulting task. If the task raises an exception it is logged
@@ -42,9 +43,9 @@ def create_task(
 def _handle_task_result(
     task: asyncio.Task[Any],
     *,
-    logger: 'LacusCoreLogAdapter',
+    logger: LacusCoreLogAdapter,
     message: str,
-    message_args: Tuple[Any, ...] = (),
+    message_args: tuple[Any, ...] = (),
 ) -> None:
     try:
         task.result()

{lacuscore-1.11.3 → lacuscore-1.12.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.11.3"
+version = "1.12.1"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -18,7 +18,6 @@ classifiers = [
     'Intended Audience :: Telecommunications Industry',
     'Intended Audience :: Information Technology',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
@@ -29,21 +28,17 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 requests = "^2.32.3"
 Sphinx = [
-   {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9,<3.10", optional = true},
    {version = "^8", python = ">=3.10", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.26.3"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.27.0"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
 redis = {version = "^5.2.0", extras = ["hiredis"]}
-dnspython = [
-    {version = "<2.7", python = "<3.9"},
-    {version = "^2.7", python = ">=3.9"}
-]
+dnspython = {version = "^2.7.0", python = ">=3.9"}
 async-timeout = {version = "^4.0.3", python = "<3.11"}
 pydantic = "^2.9.2"
 eval-type-backport = {version = "^0.2.0", python = "<3.10"}
@@ -57,7 +52,6 @@ types-redis = {version = "^4.6.0.20241004"}
 types-requests = "^2.32.0.20241016"
 types-beautifulsoup4 = "^4.12.0.20241020"
 ipython = [
-    {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]