pyproxytools 0.3.2__py3-none-any.whl

pyproxy/modules/filter.py

@@ -0,0 +1,151 @@
+"""
+pyproxy.modules.filter.py
+
+This module contains functions and a process to filter and block domains and URLs.
+It loads blocked domain names and URLs from specified files, then listens for
+incoming requests to check if the domain or URL should be blocked.
+
+Functions:
+- load_blacklist: Loads blocked FQDNs and URLs from files into sets for fast lookup.
+- filter_process: The process that checks whether a domain or URL is blocked.
+"""
+
+import multiprocessing
+import time
+import sys
+import threading
+from urllib.parse import urlparse
+import requests
+
+
+def load_blacklist(
+    blocked_sites_path: str, blocked_url_path: str, filter_mode: str
+) -> set:
+    """
+    Loads blocked FQDNs or URLs from a file or URL into a set for fast lookup.
+
+    Args:
+        blocked_sites_path (str): The path or URL to the file containing blocked FQDNs.
+        blocked_url_path (str): The path or URL to the file containing blocked URLs.
+        filter_mode (str): Mode to determine if we load from local file or HTTP URL.
+
+    Returns:
+        set: A set of blocked domains/URLs.
+    """
+    blocked_sites = set()
+    blocked_url = set()
+
+    def load_from_file(file_path: str) -> set:
+        data = set()
+        with open(file_path, "r", encoding="utf-8") as f:
+            for line in f:
+                data.add(line.strip())
+        return data
+
+    def load_from_http(url: str) -> set:
+        data = set()
+        try:
+            response = requests.get(url, timeout=3)
+            response.raise_for_status()
+            for line in response.text.splitlines():
+                data.add(line.strip())
+        except requests.exceptions.RequestException as e:
+            raise requests.exceptions.RequestException(
+                f"Failed to load data from {url}: {e}"
+            )
+        return data
+
+    if filter_mode == "local":
+        blocked_sites = load_from_file(blocked_sites_path)
+        blocked_url = load_from_file(blocked_url_path)
+    elif filter_mode == "http":
+        blocked_sites = load_from_http(blocked_sites_path)
+        blocked_url = load_from_http(blocked_url_path)
+
+    return blocked_sites, blocked_url
+
+
+def filter_process(
+    queue: multiprocessing.Queue,
+    result_queue: multiprocessing.Queue,
+    filter_mode: str,
+    blocked_sites_path: str,
+    blocked_url_path: str,
+    refresh_interval=5,
+) -> None:
+    """
+    Process that listens for requests and checks if the domain/URL should be blocked.
+
+    Args:
+        queue (multiprocessing.Queue): A queue to receive URL/domain for checking.
+        result_queue (multiprocessing.Queue): A queue to send back the result of
+                the filtering (blocked or allowed).
+        filter_mode (str): Filter list mode (local or http).
+        blocked_sites_path (str): The path to the file containing blocked FQDNs.
+        blocked_url_path (str): The path to the file containing blocked URLs.
+        refresh_interval (int): Interval in seconds to reload the blacklist files.
+    """
+    manager = multiprocessing.Manager()
+    blocked_data = manager.dict(
+        {
+            "sites": load_blacklist(blocked_sites_path, blocked_url_path, filter_mode)[
+                0
+            ],
+            "urls": load_blacklist(blocked_sites_path, blocked_url_path, filter_mode)[
+                1
+            ],
+        }
+    )
+
+    error_event = threading.Event()
+
+    def file_monitor() -> None:
+        try:
+            while True:
+                new_blocked_sites, new_blocked_url = load_blacklist(
+                    blocked_sites_path, blocked_url_path, filter_mode
+                )
+
+                blocked_data["sites"] = new_blocked_sites
+                blocked_data["urls"] = new_blocked_url
+
+                time.sleep(refresh_interval)
+        except (IOError, ValueError) as e:
+            print(f"File monitor error: {e}")
+            error_event.set()
+
+    monitor_thread = threading.Thread(target=file_monitor, daemon=True)
+    monitor_thread.start()
+
+    while True:
+        if error_event.is_set():
+            print("Error detected in file monitor thread, terminating process.")
+            sys.exit(1)
+
+        try:
+            request = queue.get()
+
+            if "://" in request:
+                parsed = urlparse(request)
+                server_host = parsed.hostname
+                url_path = parsed.path if parsed.path else "/"
+                full_url = (server_host or "") + url_path
+            else:
+                parts = request.split(":")
+                server_host = parts[0] if parts else None
+                full_url = server_host
+
+            if "*" in blocked_data["sites"] or any(
+                server_host.startswith(blocked_host)
+                for blocked_host in blocked_data["sites"]
+            ):
+                result_queue.put((server_host, "Blocked"))
+            elif any(
+                full_url.startswith(blocked_url) for blocked_url in blocked_data["urls"]
+            ):
+                result_queue.put((full_url, "Blocked"))
+            else:
+                result_queue.put((server_host, "Allowed"))
+
+        except KeyboardInterrupt:
+            break

pyproxy/modules/shortcuts.py

@@ -0,0 +1,85 @@
+"""
+pyproxy.modules.shortcuts.py
+
+This module contains functions and a process to load and manage URL shortcuts.
+It loads shortcuts (alias to URL mappings) from a specified file, and provides
+a process that listens for requests to resolve an alias to its corresponding URL.
+
+Functions:
+- load_shortcuts: Loads URL alias mappings from a file into a dictionary for fast lookup.
+- shortcuts_process: The process that listens for alias requests and resolves them to URLs.
+"""
+
+import multiprocessing
+import time
+import sys
+import threading
+
+
+def load_shortcuts(shortcuts_path: str) -> dict:
+    """
+    Loads URL alias mappings from a file into a dictionary for fast lookup.
+
+    Args:
+        shortcuts_path (str): The path to the file containing alias=URL mappings.
+
+    Returns:
+        dict: A dictionary mapping aliases to URLs.
+    """
+    shortcuts = {}
+
+    with open(shortcuts_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if "=" in line:
+                alias, url = line.split("=", 1)
+                shortcuts[alias.strip()] = url.strip()
+
+    return shortcuts
+
+
+def shortcuts_process(
+    queue: multiprocessing.Queue,
+    result_queue: multiprocessing.Queue,
+    shortcuts_path: str,
+) -> None:
+    """
+    Process that listens for alias requests and resolves them to URLs.
+
+    Args:
+        queue (multiprocessing.Queue): A queue to receive alias for URL resolution.
+        result_queue (multiprocessing.Queue): A queue to send back the resolved URL.
+        shortcuts_path (str): The path to the file containing alias=URL mappings.
+    """
+    manager = multiprocessing.Manager()
+    shortcuts_data = manager.dict({"shortcuts": load_shortcuts(shortcuts_path)})
+
+    error_event = threading.Event()
+
+    def file_monitor() -> None:
+        try:
+            while True:
+                new_shortcuts = load_shortcuts(shortcuts_path)
+
+                shortcuts_data["shortcuts"] = new_shortcuts
+
+                time.sleep(5)
+        except (IOError, ValueError) as e:
+            print(f"File monitor error: {e}")
+            error_event.set()
+
+    monitor_thread = threading.Thread(target=file_monitor, daemon=True)
+    monitor_thread.start()
+
+    while True:
+        if error_event.is_set():
+            print("Error detected in file monitor thread, terminating process.")
+            sys.exit(1)
+
+        try:
+            alias = queue.get()
+            url = shortcuts_data["shortcuts"].get(alias)
+            result_queue.put(url)
+
+        except KeyboardInterrupt:
+            break

pyproxy/monitoring/web.py

@@ -0,0 +1,279 @@
+"""
+pyproxy.monitoring.web.py
+
+This module defines a monitoring system for the ProxyServer that provides
+information about the server's processes, threads, active connections, and
+subprocesses. It includes an HTTP server implemented with Flask to expose
+monitoring endpoints for the proxy server.
+"""
+
+import os
+import threading
+import multiprocessing
+import logging
+from datetime import datetime
+from typing import List, Dict, Union
+from flask import Flask, jsonify, render_template
+from flask_httpauth import HTTPBasicAuth
+from werkzeug.security import check_password_hash, generate_password_hash
+import psutil
+
+
+def start_flask_server(proxy_server, flask_port, flask_pass, debug) -> None:
+    """
+    Starts the Flask server for monitoring the ProxyServer. It creates and
+    runs an HTTP server that exposes the proxy server's status, including
+    process and thread details, subprocess statuses, and active connections.
+
+    Args:
+        proxy_server (ProxyServer): The ProxyServer instance to monitor.
+
+    The server will expose two routes:
+        - '/' : Renders a simple index page.
+        - '/monitoring' : Returns a JSON response with the monitoring data.
+    """
+
+    class ProxyMonitor:
+        """
+        Monitors the status of the ProxyServer, including process, thread,
+        and subprocess information, as well as active client connections.
+
+        Args:
+            proxy_server (ProxyServer): The ProxyServer instance to monitor.
+        """
+
+        def __init__(self, proxy_server):
+            self.proxy_server = proxy_server
+
+        def get_process_info(
+            self,
+        ) -> Dict[str, Union[int, str, List[Dict[str, Union[int, str]]]]]:
+            """
+            Retrieves overall process information for the ProxyServer,
+            including the PID, name, status, and details about threads,
+            subprocesses, and active connections.
+
+            Returns:
+                dict: A dictionary containing the process information.
+            """
+            process_info = {
+                "pid": os.getpid(),
+                "name": "ProxyServer",
+                "status": "running",
+                "start_time": datetime.fromtimestamp(
+                    psutil.Process(os.getpid()).create_time()
+                ).strftime("%Y-%m-%d %H:%M:%S"),
+                "threads": self.get_threads_info(),
+                "subprocesses": self.get_subprocesses_info(),
+                "active_connections": self.get_active_connections(),
+            }
+            return process_info
+
+        def get_threads_info(self) -> List[Dict[str, Union[int, str]]]:
+            """
+            Retrieves information about the threads running in the ProxyServer.
+
+            Returns:
+                list: A list of dictionaries, each containing information
+                      about a thread.
+            """
+            threads_info = []
+            for thread in threading.enumerate():
+                threads_info.append(
+                    {
+                        "thread_id": thread.ident,
+                        "name": thread.name,
+                        "status": self.get_thread_status(thread),
+                    }
+                )
+            return threads_info
+
+        def get_thread_status(self, thread: threading.Thread) -> str:
+            """
+            Gets the status of a given thread.
+
+            Args:
+                thread (threading.Thread): The thread whose status is to be retrieved.
+
+            Returns:
+                str: The status of the thread ('running', 'terminated', or 'unknown').
+            """
+            try:
+                if thread.is_alive():
+                    return "running"
+                return "terminated"
+            except AttributeError:
+                return "unknown"
+
+        def get_subprocesses_info(
+            self,
+        ) -> Dict[str, Dict[str, Union[str, List[Dict[str, Union[int, str]]]]]]:
+            """
+            Retrieves the status of the ProxyServer's subprocesses, including
+            filtering, shortcuts, cancel inspection, and custom header processes.
+
+            Returns:
+                dict: A dictionary containing subprocess statuses.
+            """
+            subprocesses_info = {}
+
+            subprocesses = {
+                "filter": self.proxy_server.filter_proc,
+                "shortcuts": self.proxy_server.shortcuts_proc,
+                "cancel_inspect": self.proxy_server.cancel_inspect_proc,
+                "custom_header": self.proxy_server.custom_header_proc,
+            }
+
+            for name, process in subprocesses.items():
+                if process is not None and process.is_alive():
+                    subprocesses_info[name] = self.get_subprocess_status(process, name)
+            return subprocesses_info
+
+        def get_subprocess_status(
+            self, process: multiprocessing.Process, name: str
+        ) -> Dict[str, Union[str, None, List[Dict[str, Union[int, str]]]]]:
+            """
+            Retrieves the status of a subprocess.
+
+            Args:
+                process (multiprocessing.Process): The subprocess to check.
+                name (str): The name of the subprocess.
+
+            Returns:
+                dict: A dictionary containing the subprocess status.
+            """
+            if process is None:
+                return {"status": "not started", "name": name, "threads": []}
+            try:
+                status = "running" if process.is_alive() else "terminated"
+                threads_info = self.get_subprocess_threads_info(process)
+            except AttributeError:
+                status = "terminated"
+                threads_info = []
+            return {
+                "pid": process.pid if hasattr(process, "pid") else None,
+                "status": status,
+                "name": name,
+                "threads": threads_info,
+            }
+
+        def get_subprocess_threads_info(
+            self, process: multiprocessing.Process
+        ) -> List[Dict[str, Union[int, str]]]:
+            """
+            Retrieves the threads associated with a subprocess.
+
+            Args:
+                process (multiprocessing.Process): The subprocess to check.
+
+            Returns:
+                list: A list of dictionaries containing thread information.
+            """
+            threads_info = []
+            try:
+                for proc_thread in psutil.Process(process.pid).threads():
+                    threads_info.append(
+                        {
+                            "thread_id": proc_thread.id,
+                            "name": f"Thread-{proc_thread.id}",
+                            "status": self.get_thread_status_by_pid(proc_thread.id),
+                        }
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+            return threads_info
+
+        def get_thread_status_by_pid(self, thread_id: int) -> str:
+            """
+            Attempts to retrieve the status of a thread by its PID.
+
+            Args:
+                thread_id (int): The thread's ID.
+
+            Returns:
+                str: The status of the thread ('running' or 'terminated').
+            """
+            try:
+                process = psutil.Process(thread_id)
+                if process.is_running():
+                    return "running"
+                return "terminated"
+            except psutil.NoSuchProcess:
+                return "terminated"
+
+        def get_active_connections(self) -> List[Dict[str, Union[int, Dict]]]:
+            """
+            Retrieves information about the active client connections to the ProxyServer.
+
+            Returns:
+                list: A list of dictionaries containing information about active connections.
+            """
+            return [
+                {"thread_id": thread_id, **conn}
+                for thread_id, conn in self.proxy_server.active_connections.items()
+            ]
+
+    auth = HTTPBasicAuth()
+
+    users = {"admin": generate_password_hash(flask_pass)}
+
+    @auth.verify_password
+    def verify_password(username, password):
+        if username in users and check_password_hash(users.get(username), password):
+            return username
+        return None
+
+    app = Flask(__name__, static_folder="static")
+    if not debug:
+        log = logging.getLogger("werkzeug")
+        log.setLevel(logging.ERROR)
+
+    @app.route("/")
+    @auth.login_required
+    def index():
+        """
+        Renders the index page for the Flask application.
+
+        Returns:
+            str: The rendered HTML content of the index page.
+        """
+        return render_template("index.html")
+
+    @app.route("/monitoring", methods=["GET"])
+    @auth.login_required
+    def monitoring():
+        """
+        Returns the monitoring data for the ProxyServer in JSON format.
+
+        Returns:
+            Response: A JSON response containing the server's process information.
+        """
+        monitor = ProxyMonitor(proxy_server)
+        return jsonify(monitor.get_process_info())
+
+    @app.route("/config", methods=["GET"])
+    @auth.login_required
+    def config():
+        config_data = {
+            "host": proxy_server.host_port[0],
+            "port": proxy_server.host_port[1],
+            "debug": proxy_server.debug,
+            "html_403": proxy_server.html_403,
+            "logger_config": (
+                proxy_server.logger_config.to_dict()
+                if proxy_server.logger_config
+                else None
+            ),
+            "filter_config": (
+                proxy_server.filter_config.to_dict()
+                if proxy_server.filter_config
+                else None
+            ),
+            "ssl_config": (
+                proxy_server.ssl_config.to_dict() if proxy_server.ssl_config else None
+            ),
+            "flask_port": proxy_server.flask_port,
+        }
+        return jsonify(config_data)
+
+    app.run(host="0.0.0.0", port=flask_port)  # nosec

pyproxy/pyproxy.py

@@ -0,0 +1,107 @@
+"""
+This script implements a lightweight and fast Python-based proxy server.
+It listens for client requests, filters URLs based on a list, and allows or blocks access
+to those URLs. The proxy can handle both HTTP and HTTPS requests, and logs access and block events.
+"""
+
+from .server import ProxyServer
+from .utils.args import parse_args, load_config, get_config_value, str_to_bool
+from .utils.config import ProxyConfigLogger, ProxyConfigFilter, ProxyConfigSSL
+
+
+def main():
+    """
+    Main entry point of the proxy server. It parses command-line arguments,
+    loads the configuration file, retrieves configuration values, and starts the proxy server.
+    """
+    args = parse_args()
+    config = load_config(args.config_file)
+
+    host = get_config_value(args, config, "host", "Server", "0.0.0.0")  # nosec
+    port = int(get_config_value(args, config, "port", "Server", 8080))  # nosec
+    debug = get_config_value(args, config, "debug", "Logging", False)
+    html_403 = get_config_value(args, config, "html_403", "Files", "assets/403.html")
+    shortcuts = get_config_value(
+        args, config, "shortcuts", "Options", "config/shortcuts.txt"
+    )
+    custom_header = get_config_value(
+        args, config, "custom_header", "Options", "config/custom_header.json"
+    )
+    authorized_ips = get_config_value(
+        args, config, "authorized_ips", "Options", "config/authorized_ips.txt"
+    )
+    flask_port = get_config_value(args, config, "flask_port", "Monitoring", 5000)
+    flask_pass = get_config_value(args, config, "flask_pass", "Monitoring", "password")
+    proxy_enable = get_config_value(args, config, "proxy_enable", "Proxy", False)
+    proxy_host = get_config_value(args, config, "proxy_host", "Proxy", "127.0.0.1")
+    proxy_port = get_config_value(args, config, "proxy_port", "Proxy", 8081)
+
+    logger_config = ProxyConfigLogger(
+        access_log=get_config_value(
+            args, config, "access_log", "Logging", "logs/access.log"
+        ),
+        block_log=get_config_value(
+            args, config, "block_log", "Logging", "logs/block.log"
+        ),
+        no_logging_access=str_to_bool(
+            get_config_value(args, config, "no_logging_access", "Logging", False)
+        ),
+        no_logging_block=str_to_bool(
+            get_config_value(args, config, "no_logging_block", "Logging", False)
+        ),
+    )
+
+    filter_config = ProxyConfigFilter(
+        no_filter=str_to_bool(
+            get_config_value(args, config, "no_filter", "Filtering", False)
+        ),
+        filter_mode=get_config_value(args, config, "filter_mode", "Filtering", "local"),
+        blocked_sites=get_config_value(
+            args, config, "blocked_sites", "Filtering", "config/blocked_sites.txt"
+        ),
+        blocked_url=get_config_value(
+            args, config, "blocked_url", "Filtering", "config/blocked_url.txt"
+        ),
+    )
+
+    ssl_config = ProxyConfigSSL(
+        ssl_inspect=str_to_bool(
+            get_config_value(args, config, "ssl_inspect", "Security", False)
+        ),
+        inspect_ca_cert=get_config_value(
+            args, config, "inspect_ca_cert", "Security", "certs/ca/cert.pem"
+        ),
+        inspect_ca_key=get_config_value(
+            args, config, "inspect_ca_key", "Security", "certs/ca/key.pem"
+        ),
+        inspect_certs_folder=get_config_value(
+            args, config, "inspect_certs_folder", "Security", "certs/"
+        ),
+        cancel_inspect=get_config_value(
+            args, config, "cancel_inspect", "Security", "config/cancel_inspect.txt"
+        ),
+    )
+
+    proxy = ProxyServer(
+        host=host,
+        port=port,
+        debug=str_to_bool(debug),
+        logger_config=logger_config,
+        filter_config=filter_config,
+        ssl_config=ssl_config,
+        flask_port=flask_port,
+        flask_pass=flask_pass,
+        html_403=html_403,
+        shortcuts=shortcuts,
+        custom_header=custom_header,
+        authorized_ips=authorized_ips,
+        proxy_enable=str_to_bool(proxy_enable),
+        proxy_host=proxy_host,
+        proxy_port=proxy_port,
+    )
+
+    proxy.start()
+
+
+if __name__ == "__main__":
+    main()