lacuscore 1.17.1__tar.gz → 1.17.3__tar.gz

{lacuscore-1.17.1 → lacuscore-1.17.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.17.1
+Version: 1.17.3
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot

{lacuscore-1.17.1 → lacuscore-1.17.3}/lacuscore/lacuscore.py

@@ -9,6 +9,7 @@ import os
 import pickle
 import random
 import re
+import socket
 import sys
 import time
 import unicodedata
@@ -38,6 +39,7 @@ from redis.exceptions import DataError
 
 from . import task_logger
 from .helpers import (
+    LacusCoreException,
     LacusCoreLogAdapter, CaptureError, RetryCapture, CaptureSettingsError,
     CaptureStatus, CaptureResponse, CaptureResponseJson, CaptureSettings)
 
@@ -80,13 +82,27 @@ def _secure_filename(filename: str) -> str:
     return filename
 
 
+def _check_proxy_port_open(proxy: dict[str, str] | str) -> bool:
+    if isinstance(proxy, dict):
+        to_check = proxy['server']
+    else:
+        to_check = proxy
+    splitted_proxy_url = urlsplit(to_check)
+    if not splitted_proxy_url.hostname or not splitted_proxy_url.port:
+        raise LacusCoreException('Invalid pre-defined proxy (needs hostname and port): {proxy}')
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.settimeout(3)
+        return s.connect_ex((splitted_proxy_url.hostname, splitted_proxy_url.port)) == 0
+
+
 class LacusCore():
     """Capture URLs or web enabled documents using PlaywrightCapture.
 
     :param redis_connector: Pre-configured connector to a redis instance.
     :param max_capture_time: If the capture takes more than that time, break (in seconds)
     :param expire_results: The capture results are stored in redis. Expire them after they are done (in seconds).
-    :param tor_proxy: URL to a SOCKS 5 tor proxy. If you have tor installed, this is the default: socks5://127.0.0.1:9050.
+    :param tor_proxy: URL to a SOCKS5 tor proxy. If you have tor installed, this is the default: socks5://127.0.0.1:9050.
+    :param i2p_proxy: URL to a HTTP I2P proxy. If you have i2p installed, this is the default: http://127.0.0.1:4444.
     :param only_global_lookups: Discard captures that point to non-public IPs.
     :param max_retries: How many times should we re-try a capture if it failed.
     """
@@ -94,7 +110,8 @@ class LacusCore():
     def __init__(self, redis_connector: Redis[bytes], /, *,
                  max_capture_time: int=3600,
                  expire_results: int=36000,
-                 tor_proxy: str | None=None,
+                 tor_proxy: dict[str, str] | str | None=None,
+                 i2p_proxy: dict[str, str] | str | None=None,
                  only_global_lookups: bool=True,
                  max_retries: int=3,
                  headed_allowed: bool=False,
@@ -105,7 +122,10 @@ class LacusCore():
         self.redis = redis_connector
         self.max_capture_time = max_capture_time
         self.expire_results = expire_results
+
         self.tor_proxy = tor_proxy
+        self.i2p_proxy = i2p_proxy
+
         self.only_global_lookups = only_global_lookups
         self.max_retries = max_retries
         self.headed_allowed = headed_allowed
@@ -450,17 +470,31 @@ class LacusCore():
             proxy = to_capture.proxy
             if self.tor_proxy:
                 # check if onion or forced
-                if (proxy == 'force_tor'  # if the proxy is set to "force_tor", we use the pre-configured tor proxy, regardless the URL.
+                if (proxy == 'force_tor'  # if the proxy is set to "force_tor", we use the pre-configured tor proxy, regardless the URL, legacy feature.
                         or (not proxy  # if the TLD is "onion", we use the pre-configured tor proxy
                             and splitted_url.netloc
                             and splitted_url.hostname
                             and splitted_url.hostname.split('.')[-1] == 'onion')):
+                    if not _check_proxy_port_open(self.tor_proxy):
+                        logger.critical(f'Unable to connect to the default tor proxy: {self.tor_proxy}')
+                        raise CaptureError('The selected tor proxy is unreachable, unable to run the capture.')
                     proxy = self.tor_proxy
+                    logger.info('Using the default tor proxy.')
+            if self.i2p_proxy:
+                if (not proxy  # if the TLD is "i2p", we use the pre-configured I2P proxy
+                        and splitted_url.netloc
+                        and splitted_url.hostname
+                        and splitted_url.hostname.split('.')[-1] == 'i2p'):
+                    if not _check_proxy_port_open(self.i2p_proxy):
+                        logger.critical(f'Unable to connect to the default tor proxy: {self.i2p_proxy}')
+                        raise CaptureError('The selected I2P proxy is unreachable, unable to run the capture.')
+                    proxy = self.i2p_proxy
+                    logger.info('Using the default I2P proxy.')
 
             if self.only_global_lookups and not proxy and splitted_url.scheme not in ['data', 'file']:
                 # not relevant if we also have a proxy, or the thing to capture is a data URI or a file on disk
                 if splitted_url.netloc:
-                    if splitted_url.hostname and splitted_url.hostname.split('.')[-1] != 'onion':
+                    if splitted_url.hostname and splitted_url.hostname.split('.')[-1] not in ['onion', 'i2p']:
                         ips_to_check = []
                         # check if the hostname is an IP
                         try:

{lacuscore-1.17.1 → lacuscore-1.17.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lacuscore"
-version = "1.17.1"
+version = "1.17.3"
 description = "Core of Lacus, usable as a module"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}