fleet-framework 0.1.1__tar.gz → 0.2.0__tar.gz

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-framework
-Version: 0.1.1
+Version: 0.2.0
 Summary: generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)
 Author: Sarper Avci
 License: MIT
@@ -21,8 +21,12 @@ Requires-Dist: click>=8.1.0
 Provides-Extra: browser
 Requires-Dist: DrissionPage>=4.1.0; extra == "browser"
 Requires-Dist: cryptography>=42.0.0; extra == "browser"
+Requires-Dist: mitmproxy>=12.0; extra == "browser"
 Provides-Extra: cloudflare
 Requires-Dist: fleet-framework[browser]; extra == "cloudflare"
+Provides-Extra: cloak
+Requires-Dist: fleet-framework[browser]; extra == "cloak"
+Requires-Dist: cloakbrowser>=0.3; extra == "cloak"
 Provides-Extra: otel
 Requires-Dist: opentelemetry-api>=1.27.0; extra == "otel"
 Requires-Dist: opentelemetry-sdk>=1.27.0; extra == "otel"
@@ -31,7 +35,7 @@ Provides-Extra: test
 Requires-Dist: pytest>=8.0; extra == "test"
 Requires-Dist: pytest-asyncio>=0.23; extra == "test"
 Provides-Extra: all
-Requires-Dist: fleet-framework[browser,cloudflare,otel]; extra == "all"
+Requires-Dist: fleet-framework[browser,cloak,cloudflare,otel]; extra == "all"
 Dynamic: license-file
 
 # Fleet

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_browser/__init__.py

@@ -3,9 +3,11 @@ from fleet_browser.fingerprint import Fingerprint, FingerprintFactory
 from fleet_browser.humanizer import Humanizer
 from fleet_browser.pool import BrowserPool, slot
 from fleet_browser.proxy_extension import build_proxy_auth_extension, parse_proxy_url
+from fleet_browser.smart_router import DEFAULT_RULES, RouterStats, SmartRouter, SmartRule
 from fleet_browser.stealth import FingerprintStealth, NoOpStealth, Stealth
 
 __all__ = [
+    "DEFAULT_RULES",
     "BrowserConfig",
     "BrowserPool",
     "ChromiumWorker",
@@ -14,6 +16,9 @@ __all__ = [
     "FingerprintStealth",
     "Humanizer",
     "NoOpStealth",
+    "RouterStats",
+    "SmartRouter",
+    "SmartRule",
     "Stealth",
     "build_proxy_auth_extension",
     "parse_proxy_url",

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_browser/browser.py

@@ -13,7 +13,14 @@ from typing import Optional
 import psutil
 from DrissionPage import ChromiumOptions, ChromiumPage
 
+from fleet_browser.cloak import (
+    CLOAK_IGNORE_DEFAULT_ARGS,
+    cloak_stealth_args,
+    ensure_cloak_binary,
+    resolve_engine,
+)
 from fleet_browser.proxy_extension import build_proxy_auth_extension, parse_proxy_url
+from fleet_browser.smart_router import DEFAULT_RULES, SmartRouter, SmartRule
 from fleet_browser.stealth import Stealth
 
 logger = logging.getLogger(__name__)
@@ -120,12 +127,64 @@ class BrowserConfig:
 
     extra_args: tuple[str, ...] = ()
 
+    engine: str = "auto"
+    """Which Chromium build to launch. `"auto"` picks `"cloak"` when the
+    cloakbrowser package is installed, else falls back to `"chrome"`.
+    Set explicitly to override:
+      - `"chrome"`: use the system Chromium / Brave / Thorium auto-detect path
+      - `"cloak"`: use the CloakBrowser patched binary (downloads on first
+        use, raises if `cloakbrowser` isn't installed)
+    Cloak applies source-level canvas/WebGL/audio/font/WebRTC patches that
+    JS-level stealth can't reach. Stack FingerprintStealth on top to add
+    per-launch UA/version diversity from the uaforge corpus."""
+
+    cloak_cache_dir: Optional[str] = None
+    """Override the cloak binary cache dir (default ~/.cloakbrowser/)."""
+
+    cloak_fingerprint_seed: Optional[int] = None
+    """Force a specific cloak --fingerprint=N seed. Default: random per
+    launch. Pin a seed for reproducible tests."""
+
+    cloak_platform: str = "windows"
+    """`--fingerprint-platform` flag for cloak's binary. Must match the
+    OS family FingerprintStealth's UA claims (uaforge default is Windows).
+    Set 'macos' if you switch uaforge to a Mac corpus."""
+
+    cloak_timezone: Optional[str] = None
+    """IANA timezone for cloak's `--fingerprint-timezone` flag. Set when
+    using residential proxies so the spoofed timezone matches the egress
+    geo (otherwise CF flags the mismatch). Falls back to system tz."""
+
+    cloak_webrtc_ip: Optional[str] = None
+    """Public IP cloak should advertise via WebRTC. Set to the proxy's exit
+    IP when using residential — otherwise WebRTC leaks your real LAN IP
+    and CF correlates the mismatch."""
+
+    smart_routing: bool = True
+    """When True AND `proxy` is set, spawn a local mitmproxy that does per-flow
+    routing: only the rules-matching flows (e.g. /search) go through the paid
+    upstream, everything else goes direct. Cuts residential bandwidth ~85–95%
+    on Google/Bing/etc. without changing the SERP fetch's IP. Disable only when
+    you specifically need every byte through the proxy (rare)."""
+
+    smart_routing_rules: tuple[SmartRule, ...] = DEFAULT_RULES
+    """Per-flow routing rules. Built-ins handle Google/Bing/Yahoo/DDG /search
+    plus a block-list of trackers. Plugins can extend with additional rules
+    via tuple concat: `DEFAULT_RULES + (SmartRule(host="amazon.com", path="/s"),)`."""
+
 
 class ChromiumWorker:
 
     def __init__(self, config: BrowserConfig) -> None:
         self.config = config
         self.page: Optional[ChromiumPage] = None
+        self._smart_router: Optional[SmartRouter] = None
+
+    @property
+    def smart_router(self) -> Optional[SmartRouter]:
+        """The active SmartRouter for this worker, or None if smart routing
+        wasn't engaged. Stats are populated after `stop()` runs."""
+        return self._smart_router
         self._user_data_dir: Optional[Path] = None
         self._owns_user_data_dir = False
 
@@ -133,9 +192,16 @@ class ChromiumWorker:
         if self.page is not None:
             return self.page
 
-        binary = self.config.browser_binary or _find_browser_binary(
-            prefer=self.config.prefer_browser,
-        )
+        engine = resolve_engine(self.config.engine)
+        if engine == "cloak":
+            # Caller may still pin browser_binary explicitly — honor it.
+            binary = self.config.browser_binary or ensure_cloak_binary(
+                self.config.cloak_cache_dir
+            )
+        else:
+            binary = self.config.browser_binary or _find_browser_binary(
+                prefer=self.config.prefer_browser,
+            )
 
         self._user_data_dir = Path(tempfile.mkdtemp(prefix="chromium-worker-"))
         self._owns_user_data_dir = True
@@ -147,6 +213,23 @@ class ChromiumWorker:
         opts.set_user_data_path(str(self._user_data_dir))
         opts.set_argument(f"--window-size={self.config.window_size[0]},{self.config.window_size[1]}")
 
+        if engine == "cloak":
+            for arg in cloak_stealth_args(
+                platform=self.config.cloak_platform,
+                timezone=self.config.cloak_timezone,
+                webrtc_ip=self.config.cloak_webrtc_ip,
+                seed=self.config.cloak_fingerprint_seed,
+            ):
+                opts.set_argument(arg)
+            # Suppress Chromium defaults that leak automation signals.
+            # DrissionPage doesn't expose ignore_default_args; pass as
+            # --disable-features-replacement via plain args. The cloak
+            # binary recognises --no-enable-automation as an inversion.
+            for kill in CLOAK_IGNORE_DEFAULT_ARGS:
+                # Chromium accepts --disable-foo as the inverse of --enable-foo.
+                inv = kill.replace("--enable-", "--disable-", 1)
+                opts.set_argument(inv)
+
         # HttpsUpgrades is off so self-signed HTTPS (TokenServer) isn't rewritten.
         # Brave's ad/tracker/sync features both pollute the fingerprint and
         # sometimes break Turnstile's iframe; disable them when running Brave.
@@ -180,7 +263,26 @@ class ChromiumWorker:
             opts.set_argument("--headless=new")
 
         if self.config.proxy:
-            proxy_spec = parse_proxy_url(self.config.proxy)
+            # When smart_routing is on, spawn a SmartRouter and point Chromium
+            # at THAT instead of the upstream proxy. The router decides per-flow
+            # whether to forward upstream (paid) or direct (free). 85-95%
+            # bandwidth reduction on search engines.
+            effective_proxy = self.config.proxy
+            if self.config.smart_routing:
+                try:
+                    self._smart_router = SmartRouter(
+                        upstream_proxy=self.config.proxy,
+                        rules=self.config.smart_routing_rules,
+                    )
+                    self._smart_router.start()
+                    effective_proxy = self._smart_router.proxy_url
+                except Exception:
+                    logger.exception(
+                        "smart-router failed to start, falling back to direct upstream proxy"
+                    )
+                    self._smart_router = None
+
+            proxy_spec = parse_proxy_url(effective_proxy)
             # Chromium 122+ accepts user:pass in --proxy-server. The MV2
             # auth-extension is kept as a fallback for older Chromium.
             if proxy_spec.has_auth:
@@ -249,6 +351,13 @@ class ChromiumWorker:
                 pass
             self.page = None
 
+        if self._smart_router is not None:
+            try:
+                self._smart_router.stop()
+            except Exception:
+                logger.exception("[browser] smart-router stop failed")
+            # Keep the reference so callers can read stats after stop.
+
         # SIGKILL anything still running with our --user-data-dir; page.quit
         # sometimes leaves renderer/zygote/utility children orphaned to PID 1.
         if self._user_data_dir is not None:

fleet_framework-0.2.0/fleet_browser/cloak.py

@@ -0,0 +1,120 @@
+"""Cloak engine integration — uses the CloakBrowser patched Chromium binary.
+
+CloakBrowser is a separately-licensed binary (free for own-business use,
+OEM license required for browser-as-a-service distribution). See:
+  https://github.com/CloakHQ/CloakBrowser
+
+The wrapper Python package is MIT-licensed and pulled in via the optional
+extra: `pip install fleet-framework[cloak]`. The binary is downloaded on
+first use to ~/.cloakbrowser/ (override with CLOAKBROWSER_CACHE_DIR).
+
+Combining cloak (source-level canvas/WebGL/audio/font/WebRTC patches) with
+FingerprintStealth (CDP-level UA + Sec-CH-UA + navigator overrides) gives:
+  - Deep, JS-undetectable spoofing of low-level signals (cloak)
+  - Per-launch diversity of UA versions from the uaforge corpus (FingerprintStealth)
+
+That diversity matters: cloak's binary patches use the latest Chrome only,
+so without the CDP layer every harvest looks like the same Chrome version.
+With CDP override on top, each launch picks a different version from uaforge
+while keeping the renderer-level fingerprint coherent.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+_CLOAK_AVAILABLE: Optional[bool] = None
+
+
+def cloak_available() -> bool:
+    """Return True if the cloakbrowser wrapper package is importable.
+
+    The binary may still need to be downloaded — that's `ensure_cloak_binary`'s
+    job. This is just the cheap import probe.
+    """
+    global _CLOAK_AVAILABLE
+    if _CLOAK_AVAILABLE is None:
+        try:
+            import cloakbrowser  # noqa: F401
+            _CLOAK_AVAILABLE = True
+        except ImportError:
+            _CLOAK_AVAILABLE = False
+    return _CLOAK_AVAILABLE
+
+
+def resolve_engine(name: str) -> str:
+    """Resolve `engine="auto"` to `"cloak"` if available, else `"chrome"`."""
+    if name != "auto":
+        return name
+    return "cloak" if cloak_available() else "chrome"
+
+
+def ensure_cloak_binary(cache_dir: Optional[str] = None) -> str:
+    """Return path to the patched Chromium binary; download if missing.
+
+    Delegates to cloakbrowser.download.ensure_binary. ~250-400 MB download
+    on first use; cached at ~/.cloakbrowser/ (or `cache_dir`) thereafter.
+    Cloakbrowser checks for updates hourly when launched.
+    """
+    if not cloak_available():
+        raise RuntimeError(
+            "engine='cloak' requires cloakbrowser. "
+            "Install with: pip install fleet-framework[cloak]"
+        )
+    if cache_dir:
+        os.environ.setdefault("CLOAKBROWSER_CACHE_DIR", str(Path(cache_dir).expanduser()))
+    from cloakbrowser.download import ensure_binary
+    path = ensure_binary()
+    return str(path)
+
+
+def cloak_stealth_args(
+    *,
+    platform: str = "windows",
+    timezone: Optional[str] = None,
+    webrtc_ip: Optional[str] = None,
+    seed: Optional[int] = None,
+) -> tuple[str, ...]:
+    """Return Chromium command-line flags that activate cloak's source patches.
+
+    The `seed` drives canvas/WebGL/audio noise variation per-launch. Pass a
+    stable seed if you need reproducibility (e.g. testing). `platform` should
+    match what FingerprintStealth's UA claims — keep them aligned (default
+    'windows' matches uaforge's Windows corpus).
+    """
+    s = seed if seed is not None else random.randint(10000, 99999)
+    args = [
+        f"--fingerprint={s}",
+        f"--fingerprint-platform={platform}",
+    ]
+    if timezone:
+        args.append(f"--fingerprint-timezone={timezone}")
+    if webrtc_ip:
+        args.append(f"--fingerprint-webrtc-ip={webrtc_ip}")
+    return tuple(args)
+
+
+CLOAK_IGNORE_DEFAULT_ARGS: tuple[str, ...] = (
+    "--enable-automation",
+    "--enable-unsafe-swiftshader",
+)
+"""Default args Playwright/DrissionPage pass that leak automation signals.
+
+`--enable-automation` exposes `navigator.webdriver=true`. `--enable-unsafe-
+swiftshader` forces SwiftShader's distinctive WebGL renderer string. We pass
+these to Chromium's launch options so DrissionPage stops setting them."""
+
+
+__all__ = [
+    "CLOAK_IGNORE_DEFAULT_ARGS",
+    "cloak_available",
+    "cloak_stealth_args",
+    "ensure_cloak_binary",
+    "resolve_engine",
+]

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_browser/fingerprint.py

@@ -180,7 +180,12 @@ class FingerprintFactory:
                 "pip install git+https://github.com/sarperavci/uaforge.git"
             )
         if allowed_os is None:
-            allowed_os = (_detect_host_os(),)
+            # Default to Windows regardless of host. The host's actual OS
+            # is irrelevant — what matters is which UA + client-hint corpus
+            # the fleet's fingerprints are drawn from. Windows is the highest-
+            # entropy population (~70% of real browsing traffic) and matches
+            # the cloak engine's `--fingerprint-platform=windows` default.
+            allowed_os = ("windows",)
         self._gen = UserAgentGenerator(seed=seed)
         self._rng = random.Random(seed)
         self._min_chromium = min_chromium_version

fleet_framework-0.2.0/fleet_browser/smart_router.py

@@ -0,0 +1,354 @@
+"""Smart per-flow proxy router for browser-based automation.
+
+The problem this solves: residential proxy bandwidth costs $0.50-1.00/GB.
+A vanilla Chromium load of a Google SERP through a residential proxy burns
+5+ MB per query (Google's own ML model downloads, gstatic CDN assets, JS
+bundles). Only the search-result document actually needs the residential
+IP — Google's bot-wall scores the document fetch, not its subresources.
+
+How it works: a local mitmproxy subprocess acts as Chromium's proxy. Per-flow
+the addon decides:
+
+  - HOST + PATH matches a "paid" rule → forward through the upstream
+    residential proxy → captures the IP-reputation signal
+  - Otherwise → flow.server_conn.via = None → forward direct → free, doesn't
+    affect the SERP fetch
+  - Matches a "block" rule → return 204 immediately
+
+In our measurements: 1.31 MB → 0.17 MB paid per Google query. 87% reduction.
+
+ChromiumWorker spins up a SmartRouter automatically when `BrowserConfig.proxy`
+is set AND `smart_routing=True` (the default). Per-process subprocess. Stats
+(paid/direct/blocked bytes + counts) are written to a temp file and read on
+stop, so callers can attribute cost per task.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import signal
+import socket
+import subprocess
+import tempfile
+import textwrap
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SmartRule:
+    """One routing rule. Fields are AND-ed when both are set.
+
+    `host_match` is matched as a SUFFIX so `google.com` covers www.google.com,
+    images.google.com, etc. Use a leading "." for exact subdomain ("google.com"
+    matches "google.com" but ".google.com" matches "*.google.com" only).
+
+    `path_prefix` is a literal prefix on the URL path. Empty = match any path.
+
+    `action`:
+      - "paid"   → forward through the upstream residential proxy
+      - "direct" → forward direct, bypassing the upstream
+      - "block"  → drop the request (returns 204; useful for trackers/ads)
+    """
+
+    host_match: str
+    path_prefix: str = ""
+    action: str = "paid"
+
+
+DEFAULT_RULES: tuple[SmartRule, ...] = (
+    SmartRule(host_match="www.google.com", path_prefix="/search", action="paid"),
+    SmartRule(host_match="www.bing.com", path_prefix="/search", action="paid"),
+    SmartRule(host_match="search.yahoo.com", path_prefix="/search", action="paid"),
+    SmartRule(host_match="duckduckgo.com", path_prefix="/?q", action="paid"),
+    SmartRule(host_match="duckduckgo.com", path_prefix="/html", action="paid"),
+    # Block known-noise endpoints we never want to fetch (also free, so the
+    # blocking saves on direct bandwidth + latency, not paid cost).
+    SmartRule(host_match="googletagmanager.com", action="block"),
+    SmartRule(host_match="googlesyndication.com", action="block"),
+    SmartRule(host_match="googleadservices.com", action="block"),
+    SmartRule(host_match="doubleclick.net", action="block"),
+    SmartRule(host_match="google-analytics.com", action="block"),
+    SmartRule(host_match="optimizationguide-pa.googleapis.com", action="block"),
+)
+
+
+@dataclass
+class RouterStats:
+    paid_bytes: int = 0
+    paid_requests: int = 0
+    direct_bytes: int = 0
+    direct_requests: int = 0
+    blocked_requests: int = 0
+
+
+def _pick_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def _is_port_open(host: str, port: int) -> bool:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.settimeout(0.15)
+        try:
+            s.connect((host, port))
+            return True
+        except (socket.timeout, ConnectionRefusedError, OSError):
+            return False
+
+
+def _find_mitmdump() -> Optional[str]:
+    """Locate the mitmdump binary. Returns None if not installed."""
+    p = shutil.which("mitmdump")
+    if p:
+        return p
+    # mitmproxy is often pip-installed under the venv's bin
+    here = Path(__file__).resolve()
+    for candidate in (
+        here.parents[2] / ".venv" / "bin" / "mitmdump",
+        Path.home() / ".local" / "bin" / "mitmdump",
+    ):
+        if candidate.exists() and os.access(candidate, os.X_OK):
+            return str(candidate)
+    return None
+
+
+@dataclass
+class SmartRouter:
+    """Embedded mitmproxy subprocess that does per-flow upstream routing.
+
+    Lifecycle: start() spawns mitmdump, blocks until its listen port is open;
+    stop() sends SIGTERM and reads the stats file. Designed for one router
+    per ChromiumWorker — cheap to spin up (~150ms cold start).
+    """
+
+    upstream_proxy: str
+    """Upstream residential proxy URL with auth: http://user:pass@host:port."""
+
+    rules: tuple[SmartRule, ...] = field(default_factory=lambda: DEFAULT_RULES)
+    """Routing rules. First matching rule wins. Default for non-matches: direct."""
+
+    default_action: str = "direct"
+    """Action when no rule matches. "direct" = free, "paid" = everything paid."""
+
+    listen_host: str = "127.0.0.1"
+    listen_port: int = 0
+    """0 = auto-pick free port."""
+
+    _proc: Optional[subprocess.Popen] = field(default=None, repr=False, init=False)
+    _stats_path: Optional[Path] = field(default=None, repr=False, init=False)
+    _addon_path: Optional[Path] = field(default=None, repr=False, init=False)
+    _stats: RouterStats = field(default_factory=RouterStats, init=False)
+
+    @property
+    def proxy_url(self) -> str:
+        return f"http://{self.listen_host}:{self.listen_port}"
+
+    @property
+    def stats(self) -> RouterStats:
+        return self._stats
+
+    def start(self, *, startup_timeout: float = 10.0) -> None:
+        mitmdump = _find_mitmdump()
+        if mitmdump is None:
+            raise RuntimeError(
+                "SmartRouter requires mitmproxy. "
+                "Install with: pip install 'fleet-framework[browser]'"
+            )
+        if self.listen_port == 0:
+            self.listen_port = _pick_free_port()
+
+        # Strip auth from upstream URL and pass separately.
+        from urllib.parse import urlparse
+        u = urlparse(self.upstream_proxy)
+        upstream_clean = f"{u.scheme}://{u.hostname}:{u.port or 80}"
+        upstream_auth = f"{u.username}:{u.password}" if u.username else None
+
+        # Persist stats to a tmp file the subprocess writes to.
+        self._stats_path = Path(tempfile.mktemp(prefix="smart-router-stats-", suffix=".json"))
+        # Generate the addon script — embeds the rules so the subprocess has them.
+        rules_repr = [asdict(r) for r in self.rules]
+        self._addon_path = Path(tempfile.mktemp(prefix="smart-router-addon-", suffix=".py"))
+        self._addon_path.write_text(_render_addon_script(
+            rules_repr, self.default_action, str(self._stats_path)
+        ))
+
+        cmd = [
+            mitmdump,
+            "-p", str(self.listen_port),
+            "--mode", f"upstream:{upstream_clean}",
+            "--set", "ssl_insecure=true",
+            "-s", str(self._addon_path),
+            "--quiet",
+        ]
+        if upstream_auth:
+            cmd.extend(["--upstream-auth", upstream_auth])
+
+        self._proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            start_new_session=True,
+        )
+        # Wait for the listen socket.
+        deadline = time.monotonic() + startup_timeout
+        while time.monotonic() < deadline:
+            if _is_port_open(self.listen_host, self.listen_port):
+                logger.info(
+                    "smart-router @ %s, %d rules, upstream=%s",
+                    self.proxy_url, len(self.rules), upstream_clean,
+                )
+                return
+            if self._proc.poll() is not None:
+                err = (self._proc.stderr.read() if self._proc.stderr else b"").decode(errors="replace")
+                self._cleanup_files()
+                raise RuntimeError(f"SmartRouter mitmdump exited early: {err[:400]}")
+            time.sleep(0.1)
+        self.stop()
+        raise RuntimeError(f"SmartRouter failed to start within {startup_timeout}s")
+
+    def stop(self) -> None:
+        # Read stats BEFORE killing the process — the addon flushes on every
+        # response, so the file is current up to the last completed flow.
+        self._read_stats()
+        if self._proc is not None:
+            try:
+                # Term the whole process group; mitmdump spawns helper threads.
+                os.killpg(os.getpgid(self._proc.pid), signal.SIGTERM)
+            except (ProcessLookupError, PermissionError, OSError):
+                pass
+            try:
+                self._proc.wait(timeout=3)
+            except subprocess.TimeoutExpired:
+                try:
+                    os.killpg(os.getpgid(self._proc.pid), signal.SIGKILL)
+                except (ProcessLookupError, OSError):
+                    pass
+                self._proc.wait(timeout=2)
+            self._proc = None
+        # Re-read stats one more time in case any flows finished between the
+        # first read and the SIGTERM.
+        self._read_stats()
+        self._cleanup_files()
+        logger.info(
+            "smart-router stopped. paid=%d B / %d req, direct=%d B / %d req, blocked=%d req",
+            self._stats.paid_bytes, self._stats.paid_requests,
+            self._stats.direct_bytes, self._stats.direct_requests,
+            self._stats.blocked_requests,
+        )
+
+    def __enter__(self) -> "SmartRouter":
+        self.start()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.stop()
+
+    def _read_stats(self) -> None:
+        if self._stats_path is None or not self._stats_path.exists():
+            return
+        try:
+            data = json.loads(self._stats_path.read_text() or "{}")
+            self._stats = RouterStats(
+                paid_bytes=int(data.get("paid_bytes", 0)),
+                paid_requests=int(data.get("paid_requests", 0)),
+                direct_bytes=int(data.get("direct_bytes", 0)),
+                direct_requests=int(data.get("direct_requests", 0)),
+                blocked_requests=int(data.get("blocked_requests", 0)),
+            )
+        except (json.JSONDecodeError, ValueError, OSError):
+            logger.debug("smart-router stats file unreadable", exc_info=True)
+
+    def _cleanup_files(self) -> None:
+        for p in (self._stats_path, self._addon_path):
+            if p is not None and p.exists():
+                try:
+                    p.unlink()
+                except OSError:
+                    pass
+
+
+def _render_addon_script(rules: list[dict], default_action: str, stats_path: str) -> str:
+    """Generate the mitmproxy addon script. Embeds the rules as a literal so
+    the subprocess has them at startup without a separate config file."""
+    return textwrap.dedent(f"""
+        import json
+        from mitmproxy import http
+        from pathlib import Path
+
+        RULES = {json.dumps(rules)}
+        DEFAULT_ACTION = {default_action!r}
+        STATS_PATH = {stats_path!r}
+
+        _stats = {{
+            "paid_bytes": 0, "paid_requests": 0,
+            "direct_bytes": 0, "direct_requests": 0,
+            "blocked_requests": 0,
+        }}
+
+        def _host_matches(host, pattern):
+            return host == pattern or host.endswith("." + pattern) or host.endswith(pattern)
+
+        def _pick_action(host, path):
+            for r in RULES:
+                if not _host_matches(host, r["host_match"]):
+                    continue
+                if r["path_prefix"] and not path.startswith(r["path_prefix"]):
+                    continue
+                return r["action"]
+            return DEFAULT_ACTION
+
+        def _persist():
+            try:
+                Path(STATS_PATH).write_text(json.dumps(_stats))
+            except OSError:
+                pass
+
+        def request(flow):
+            host = flow.request.pretty_host
+            path = flow.request.path
+            action = _pick_action(host, path)
+            flow.metadata["smart_action"] = action
+            if action == "block":
+                _stats["blocked_requests"] += 1
+                flow.response = http.Response.make(
+                    204, b"", {{"x-smart-router": "blocked"}}
+                )
+                _persist()
+                return
+            if action == "direct" and flow.server_conn.via is not None:
+                try:
+                    flow.server_conn.via = None
+                except Exception:
+                    pass
+
+        def response(flow):
+            action = flow.metadata.get("smart_action", "direct")
+            body = flow.response.raw_content or b""
+            resp_h = sum(len(k) + len(v) + 4 for k, v in flow.response.headers.items())
+            req_h = sum(len(k) + len(v) + 4 for k, v in flow.request.headers.items())
+            wire = len(body) + resp_h + req_h + len(flow.request.raw_content or b"")
+            if action == "paid":
+                _stats["paid_bytes"] += wire
+                _stats["paid_requests"] += 1
+            elif action == "direct":
+                _stats["direct_bytes"] += wire
+                _stats["direct_requests"] += 1
+            _persist()
+    """).strip()
+
+
+__all__ = [
+    "DEFAULT_RULES",
+    "RouterStats",
+    "SmartRouter",
+    "SmartRule",
+]

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_framework.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-framework
-Version: 0.1.1
+Version: 0.2.0
 Summary: generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)
 Author: Sarper Avci
 License: MIT
@@ -21,8 +21,12 @@ Requires-Dist: click>=8.1.0
 Provides-Extra: browser
 Requires-Dist: DrissionPage>=4.1.0; extra == "browser"
 Requires-Dist: cryptography>=42.0.0; extra == "browser"
+Requires-Dist: mitmproxy>=12.0; extra == "browser"
 Provides-Extra: cloudflare
 Requires-Dist: fleet-framework[browser]; extra == "cloudflare"
+Provides-Extra: cloak
+Requires-Dist: fleet-framework[browser]; extra == "cloak"
+Requires-Dist: cloakbrowser>=0.3; extra == "cloak"
 Provides-Extra: otel
 Requires-Dist: opentelemetry-api>=1.27.0; extra == "otel"
 Requires-Dist: opentelemetry-sdk>=1.27.0; extra == "otel"
@@ -31,7 +35,7 @@ Provides-Extra: test
 Requires-Dist: pytest>=8.0; extra == "test"
 Requires-Dist: pytest-asyncio>=0.23; extra == "test"
 Provides-Extra: all
-Requires-Dist: fleet-framework[browser,cloudflare,otel]; extra == "all"
+Requires-Dist: fleet-framework[browser,cloak,cloudflare,otel]; extra == "all"
 Dynamic: license-file
 
 # Fleet

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_framework.egg-info/SOURCES.txt

@@ -42,10 +42,12 @@ fleet/worker/ws_client.py
 fleet_browser/__init__.py
 fleet_browser/browser.py
 fleet_browser/cert.py
+fleet_browser/cloak.py
 fleet_browser/fingerprint.py
 fleet_browser/humanizer.py
 fleet_browser/pool.py
 fleet_browser/proxy_extension.py
+fleet_browser/smart_router.py
 fleet_browser/solver.py
 fleet_browser/stealth.py
 fleet_cloudflare/__init__.py

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/fleet_framework.egg-info/requires.txt

@@ -9,11 +9,16 @@ psutil>=5.9.0
 click>=8.1.0
 
 [all]
-fleet-framework[browser,cloudflare,otel]
+fleet-framework[browser,cloak,cloudflare,otel]
 
 [browser]
 DrissionPage>=4.1.0
 cryptography>=42.0.0
+mitmproxy>=12.0
+
+[cloak]
+fleet-framework[browser]
+cloakbrowser>=0.3
 
 [cloudflare]
 fleet-framework[browser]

{fleet_framework-0.1.1 → fleet_framework-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fleet-framework"
-version = "0.1.1"
+version = "0.2.0"
 description = "generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -26,10 +26,15 @@ dependencies = [
 browser = [
     "DrissionPage>=4.1.0",
     "cryptography>=42.0.0",
+    "mitmproxy>=12.0",  # SmartRouter subprocess for per-flow proxy routing
 ]
 cloudflare = [
     "fleet-framework[browser]",
 ]
+cloak = [
+    "fleet-framework[browser]",
+    "cloakbrowser>=0.3",
+]
 otel = [
     "opentelemetry-api>=1.27.0",
     "opentelemetry-sdk>=1.27.0",
@@ -40,7 +45,7 @@ test = [
     "pytest-asyncio>=0.23",
 ]
 all = [
-    "fleet-framework[browser,cloudflare,otel]",
+    "fleet-framework[browser,cloudflare,cloak,otel]",
 ]
 
 [project.scripts]