fleet-framework 0.2.0__tar.gz → 0.3.0__tar.gz

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-framework
-Version: 0.2.0
+Version: 0.3.0
 Summary: generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)
 Author: Sarper Avci
 License: MIT
@@ -19,14 +19,15 @@ Requires-Dist: jinja2>=3.1.0
 Requires-Dist: psutil>=5.9.0
 Requires-Dist: click>=8.1.0
 Provides-Extra: browser
-Requires-Dist: DrissionPage>=4.1.0; extra == "browser"
+Requires-Dist: cloakbrowser>=0.3; extra == "browser"
 Requires-Dist: cryptography>=42.0.0; extra == "browser"
-Requires-Dist: mitmproxy>=12.0; extra == "browser"
+Requires-Dist: mitmproxy>=11.0; extra == "browser"
+Requires-Dist: curl_cffi>=0.7; extra == "browser"
+Requires-Dist: pyvirtualdisplay>=3.0; extra == "browser"
 Provides-Extra: cloudflare
 Requires-Dist: fleet-framework[browser]; extra == "cloudflare"
 Provides-Extra: cloak
 Requires-Dist: fleet-framework[browser]; extra == "cloak"
-Requires-Dist: cloakbrowser>=0.3; extra == "cloak"
 Provides-Extra: otel
 Requires-Dist: opentelemetry-api>=1.27.0; extra == "otel"
 Requires-Dist: opentelemetry-sdk>=1.27.0; extra == "otel"

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/cli.py

@@ -249,7 +249,7 @@ def describe_cmd(automation_type: str) -> None:
     click.echo(json.dumps(doc["config"], indent=2))
     if "queue" in doc:
         click.echo("")
-        click.echo(f"task payload schema (the shape -k / -p builds):")
+        click.echo("task payload schema (the shape -k / -p builds):")
         click.echo(json.dumps(doc["queue"]["payload"], indent=2))
     if "stream" in doc:
         click.echo("")

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/core/automation.py

@@ -7,6 +7,7 @@ from typing import Any, ClassVar, Generic, Optional, TypeVar
 
 from pydantic import BaseModel
 
+from fleet.core.browser import BrowserNeeds
 from fleet.core.config import BaseConfig
 from fleet.core.context import Context
 from fleet.core.contract import Pool, Queue, Stream
@@ -33,6 +34,7 @@ class BaseAutomation(Generic[C]):
     Output: ClassVar[Optional[type[BaseModel]]] = None      # stream payload type (one stream per automation, by convention)
     TaskPayload: ClassVar[Optional[type[BaseModel]]] = None # BatchAutomation task payload
     Pools: ClassVar[dict[str, Pool]] = {}                   # pools this automation owns
+    Browser: ClassVar[Optional[BrowserNeeds]] = None        # opt-in browser-pool requirement; framework provisions ctx.browser
 
     automation_type: ClassVar[str] = ""
 

fleet_framework-0.3.0/fleet/core/browser.py

@@ -0,0 +1,292 @@
+"""Browser pool as a first-class core resource.
+
+`BrowserNeeds` is the declarative side: an automation declares what kind of
+browser-pool it needs at the class level, the same way it declares
+`TaskPayload`, `Output`, and `Pools`. The worker bootstrap reads these
+declarations, provisions one shared `BrowserResource` per unique need, and
+injects it as `ctx.browser`.
+
+Plugins should never `import fleet_browser` directly. They:
+
+    class MyAutomation(BatchAutomation):
+        Browser = BrowserNeeds(size=4, max_uses=1, engine="cloak")
+
+        async def run_one(self, task, ctx):
+            async with ctx.browser.acquire(proxy=ctx.proxy_url) as page:
+                await page.goto(...)
+
+`fleet_browser` stays an optional install — `BrowserResource.start()` only
+imports it when an automation actually declares a `Browser` need.
+
+Design notes:
+
+* The Browser is pooled and pre-warmed at worker boot; SmartRouter is per-
+  acquire because each task usually has its own proxy URL (residential
+  sticky-session rotation). mitmdump can't be reconfigured mid-flight, so
+  pooling it doesn't help — a fresh router per task is cheap (~150 ms).
+
+* Each acquire creates a new Playwright BrowserContext so cookies / storage
+  / page state from a previous task can't leak into the next. The cloak
+  fingerprint stays at the Browser-launch level, so fingerprint refresh per
+  task requires `max_uses=1` (dispose + respawn the Browser between uses).
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class BrowserNeeds:
+    """Declarative browser-pool requirement attached to an automation class.
+
+    The worker boots one ``BrowserResource`` per *unique* ``BrowserNeeds``
+    across all loaded automations — automations that declare identical
+    needs share a pool.
+
+    Attributes
+    ----------
+    size : int
+        Pool size — number of pre-warmed Browsers kept hot. Lease is
+        instant while at least one slot is idle.
+
+    max_uses : int
+        How many acquires one Browser handles before being recycled.
+        ``1`` = fresh cloak fingerprint per task (recommended against
+        Google-class bot walls). Larger values reuse the Browser across
+        many tasks, with cookies cleared between (cheaper but the cloak
+        seed persists).
+
+    max_age_seconds : float
+        Hard recycling window. A Browser past this age is disposed even
+        if its use count is below ``max_uses`` — defends against process
+        memory leaks.
+
+    engine : str
+        Browser engine. ``"auto"`` resolves to ``"cloak"`` when
+        cloakbrowser is installed, ``"chrome"`` is stock Playwright
+        Chromium.
+
+    headless : bool
+        Run headless. Default ``False`` because headed-on-Xvfb beats
+        headless for anti-bot fingerprint stability; the framework
+        auto-starts a managed Xvfb so the deployment doesn't need to.
+
+    smart_routing : bool
+        When ``True`` and the caller passes ``proxy=...`` to ``acquire``,
+        a per-task SmartRouter is spun up so paid flows go via curl_cffi
+        with a real Chrome JA3. Set ``False`` to bypass the router and
+        use the proxy directly (cheaper but loses CDN-family JA3 coherence).
+    """
+
+    size: int = 4
+    max_uses: int = 1
+    max_age_seconds: float = 3600.0
+    engine: str = "auto"
+    headless: bool = False
+    smart_routing: bool = True
+
+    def __post_init__(self) -> None:
+        if self.size <= 0:
+            raise ValueError("BrowserNeeds.size must be >= 1")
+        if self.max_uses <= 0:
+            raise ValueError("BrowserNeeds.max_uses must be >= 1")
+        if self.max_age_seconds <= 0:
+            raise ValueError("BrowserNeeds.max_age_seconds must be > 0")
+        if self.engine not in ("auto", "cloak", "chrome"):
+            raise ValueError(
+                f"BrowserNeeds.engine must be one of auto/cloak/chrome, "
+                f"got {self.engine!r}"
+            )
+
+
+@dataclass
+class BrowserResource:
+    """Process-wide handle wrapping a pre-warmed BrowserPool.
+
+    Constructed by the worker bootstrap (one per unique BrowserNeeds),
+    started before any slot runs, stopped on worker shutdown. Plugins
+    access it via ``ctx.browser`` and call ``.acquire(proxy=...)`` per
+    task.
+    """
+
+    needs: BrowserNeeds
+    _pool: Any = field(default=None, init=False, repr=False)  # BrowserPool, lazy import
+    _started: bool = field(default=False, init=False, repr=False)
+
+    async def start(self) -> None:
+        """Pre-warm the underlying BrowserPool.
+
+        Imports fleet_browser lazily so the core stays importable in
+        deployments that don't need browser automation.
+        """
+        if self._started:
+            return
+        try:
+            from fleet_browser import BrowserConfig, make_browser_pool
+        except ImportError as e:
+            raise RuntimeError(
+                "BrowserNeeds declared but fleet_browser is not installed. "
+                "Install with: pip install fleet-framework[browser]"
+            ) from e
+
+        template = BrowserConfig(
+            page_url="about:blank",
+            proxy=None,  # Browser launches without proxy; per-acquire context sets it
+            engine=self.needs.engine,
+            headless=self.needs.headless,
+            smart_routing=False,  # we manage SmartRouter per-acquire
+        )
+        self._pool = make_browser_pool(
+            size=self.needs.size,
+            config_template=template,
+            max_uses=self.needs.max_uses,
+            max_age_seconds=self.needs.max_age_seconds,
+        )
+        await self._pool.start()
+        self._started = True
+        logger.info(
+            "browser-resource started: size=%d max_uses=%d engine=%s",
+            self.needs.size, self.needs.max_uses, self.needs.engine,
+        )
+
+    async def stop(self) -> None:
+        """Dispose every pooled Browser. Idempotent."""
+        if not self._started:
+            return
+        try:
+            await self._pool.stop()
+        finally:
+            self._started = False
+            self._pool = None
+            logger.info("browser-resource stopped")
+
+    @asynccontextmanager
+    async def acquire(
+        self, *, proxy: Optional[str] = None,
+    ) -> AsyncIterator["Page"]:
+        """Lease a Page bound to this task's proxy.
+
+        Implementation: leases a Browser from the pool, spins up a
+        per-task SmartRouter (if ``smart_routing`` and a proxy was
+        provided), creates a fresh BrowserContext pointed at the router,
+        yields a new Page. On exit the context is closed and the router
+        stopped; the Browser returns to the pool (or is disposed per
+        ``max_uses``).
+        """
+        if not self._started:
+            raise RuntimeError("BrowserResource not started")
+
+        async with self._pool.acquire() as base_page:
+            # base_page is the launch-time page from the pool entry; we don't
+            # use it directly because it lives in a shared context. New
+            # context per acquire = clean cookie/storage isolation.
+            browser = base_page.context.browser
+            if browser is None:
+                raise RuntimeError("pooled page has no parent browser")
+
+            router = None
+            proxy_settings: Optional[dict] = None
+            ignore_https = False
+            if proxy and self.needs.smart_routing:
+                from fleet_browser.smart_router import SmartRouter
+                router = SmartRouter(upstream_proxy=proxy)
+                router.start()
+                proxy_settings = {"server": router.proxy_url}
+                ignore_https = True  # browser must trust mitm's self-signed CA
+            elif proxy:
+                proxy_settings = _proxy_settings_from_url(proxy)
+
+            new_ctx = await browser.new_context(
+                proxy=proxy_settings,
+                ignore_https_errors=ignore_https,
+            )
+            page = await new_ctx.new_page()
+            # Expose the router on the page so callers can read stats /
+            # families without having to thread it through.
+            try:
+                page._smart_router = router  # type: ignore[attr-defined]
+            except Exception:
+                pass
+            try:
+                yield page
+            finally:
+                try:
+                    await new_ctx.close()
+                except Exception:
+                    logger.debug("context close failed", exc_info=True)
+                if router is not None:
+                    try:
+                        router.stop()
+                    except Exception:
+                        logger.debug("router stop failed", exc_info=True)
+
+
+def _proxy_settings_from_url(url: str) -> dict:
+    """Convert ``http://user:pass@host:port`` into Playwright proxy settings."""
+    from urllib.parse import urlparse
+    u = urlparse(url)
+    server = f"{u.scheme}://{u.hostname}:{u.port or 80}"
+    out: dict[str, str] = {"server": server}
+    if u.username:
+        out["username"] = u.username
+        out["password"] = u.password or ""
+    return out
+
+
+class BrowserRegistry:
+    """Holds one BrowserResource per unique BrowserNeeds across all loaded
+    automations. Built once at worker boot, queried by SlotRunner to attach
+    the right resource to each Context."""
+
+    def __init__(self) -> None:
+        self._resources: dict[BrowserNeeds, BrowserResource] = {}
+        self._started: bool = False
+
+    def register(self, needs: BrowserNeeds) -> BrowserResource:
+        """Return the shared resource for these needs, creating one if needed.
+        Must be called before ``start_all``."""
+        if needs in self._resources:
+            return self._resources[needs]
+        res = BrowserResource(needs=needs)
+        self._resources[needs] = res
+        return res
+
+    async def start_all(self) -> None:
+        """Pre-warm every registered resource in parallel."""
+        if self._started:
+            return
+        if not self._resources:
+            self._started = True
+            return
+        await asyncio.gather(*(r.start() for r in self._resources.values()))
+        self._started = True
+
+    async def stop_all(self) -> None:
+        """Stop every resource. Idempotent."""
+        if not self._started:
+            return
+        await asyncio.gather(
+            *(r.stop() for r in self._resources.values()),
+            return_exceptions=True,
+        )
+        self._started = False
+        self._resources.clear()
+
+    def __len__(self) -> int:
+        return len(self._resources)
+
+
+__all__ = [
+    "BrowserNeeds",
+    "BrowserRegistry",
+    "BrowserResource",
+]

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/core/context.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel
 
 if TYPE_CHECKING:
     from fleet.core.backend import Backend
+    from fleet.core.browser import BrowserResource
     from fleet.core.config import BaseConfig
     from fleet.core.contract import Pool, Queue, Stream
     from fleet.core.events import EventBus
@@ -37,6 +38,7 @@ class Context(Generic[C]):
     _automation_cls: Any = None  # type[BaseAutomation], set by SlotRunner
     _proxy: Optional["ProxyHandle"] = None
     proxy_url: Optional[str] = None
+    browser: Optional["BrowserResource"] = None  # set when automation declares Browser
     extra: dict[str, Any] = field(default_factory=dict)
 
     # ---- proxy ----

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/core/local_runner.py

@@ -29,6 +29,7 @@ from fleet.core.automation import (
     ContinuousAutomation,
     Task,
 )
+from fleet.core.browser import BrowserRegistry
 from fleet.core.context import Context
 from fleet.core.memory_backend import InMemoryBackend
 from fleet.core.metrics import SlotMetrics
@@ -72,8 +73,12 @@ def _build_context(
     log: logging.Logger,
     worker_id: str,
     slot_id: int = 0,
+    browser_registry: Optional[BrowserRegistry] = None,
 ) -> tuple[Context, asyncio.Event]:
     shutdown = asyncio.Event()
+    browser_resource = None
+    if browser_registry is not None and cls.Browser is not None:
+        browser_resource = browser_registry.register(cls.Browser)
     ctx = Context(
         automation_type=cls.automation_type,
         worker_id=worker_id,
@@ -86,6 +91,7 @@ def _build_context(
         _events=_StubEventBus(),  # type: ignore[arg-type]
         _metrics=SlotMetrics(),
         _automation_cls=cls,
+        browser=browser_resource,
     )
     return ctx, shutdown
 
@@ -124,7 +130,19 @@ async def run_local(
     ]
 
     emit_cb = emit or _default_emit
-    ctx, _shutdown = _build_context(cls, cfg, emit_cb, backend, log, worker_id)
+
+    # Provision the browser pool BEFORE building the context so ctx.browser
+    # is ready by the time run_one() fires. Plugins that don't declare
+    # Browser get a noop registry and ctx.browser stays None.
+    browser_registry = BrowserRegistry()
+    if cls.Browser is not None:
+        browser_registry.register(cls.Browser)
+    await browser_registry.start_all()
+
+    ctx, _shutdown = _build_context(
+        cls, cfg, emit_cb, backend, log, worker_id,
+        browser_registry=browser_registry,
+    )
 
     sem = asyncio.Semaphore(max(1, concurrency))
     ok = 0
@@ -154,6 +172,7 @@ async def run_local(
             await automation.cleanup(ctx)
         except Exception:
             log.exception("cleanup() raised")
+        await browser_registry.stop_all()
         await backend.aclose()
 
     return {"ok": ok, "failed": failed}
@@ -193,8 +212,16 @@ async def run_local_continuous(
         emit_count += 1
         await (emit or _default_emit)(payload)
 
+    browser_registry = BrowserRegistry()
+    if cls.Browser is not None:
+        browser_registry.register(cls.Browser)
+    await browser_registry.start_all()
+
     contexts = [
-        _build_context(cls, cfg, _counting_emit, backend, log, worker_id, slot_id=i)
+        _build_context(
+            cls, cfg, _counting_emit, backend, log, worker_id,
+            slot_id=i, browser_registry=browser_registry,
+        )
         for i in range(max(1, slots))
     ]
 
@@ -226,6 +253,7 @@ async def run_local_continuous(
             await automation.cleanup(contexts[0][0])
         except Exception:
             log.exception("cleanup() raised")
+        await browser_registry.stop_all()
         await backend.aclose()
 
     return {"emits": emit_count}

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/worker/agent.py

@@ -12,6 +12,7 @@ import httpx
 import psutil
 
 from fleet.core.automation import BaseAutomation, load_entry_points
+from fleet.core.browser import BrowserRegistry
 from fleet.core.protocol import ConfigChanged, Drain, Hardware
 from fleet.worker.reconcile_loop import ReconcileLoop
 from fleet.worker.slot_runner import SlotRunner
@@ -56,13 +57,24 @@ class Agent:
         self._slot_runner: SlotRunner | None = None
         self._stop = asyncio.Event()
 
+        # Shared resources for plugins to consume via ctx. Browser pool is
+        # the only one for now; future ones (HTTP session, cache, etc.) can
+        # follow the same opt-in declaration pattern on BaseAutomation.
+        self._browser_registry = BrowserRegistry()
+        if self._auto.Browser is not None:
+            self._browser_registry.register(self._auto.Browser)
+
     async def run(self) -> None:
         ws_url = (
             self._master_url.replace("http://", "ws://").replace("https://", "wss://")
             + f"/api/v1/automations/{self._auto.automation_type}/workers/{self._worker_id}/control"
         )
+        # Pre-warm shared resources BEFORE accepting slot work.
+        await self._browser_registry.start_all()
+
         self._slot_runner = SlotRunner(
-            self._auto, self._worker_id, self._redis_url, self._on_output
+            self._auto, self._worker_id, self._redis_url, self._on_output,
+            browser_registry=self._browser_registry,
         )
         self._reconcile = ReconcileLoop(self._slot_runner.make_task, self._cfg_cls)
 
@@ -88,6 +100,7 @@ class Agent:
                 await self._reconcile.stop()
             if self._ws is not None:
                 await self._ws.stop()
+            await self._browser_registry.stop_all()
 
     async def stop(self) -> None:
         self._stop.set()

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet/worker/slot_runner.py

@@ -14,6 +14,7 @@ from fleet.core.automation import (
     Task,
 )
 from fleet.core.backend import Backend, backend_from_url
+from fleet.core.browser import BrowserRegistry
 from fleet.core.context import Context
 from fleet.core.events import EventBus
 from fleet.core.logging import bind_log_context, reset_log_context
@@ -39,6 +40,8 @@ class SlotRunner:
         worker_id: str,
         redis_url: str,
         on_output: Any,  # async (slot_id, payload_dict) -> None
+        *,
+        browser_registry: Optional[BrowserRegistry] = None,
     ) -> None:
         self._auto = automation
         self._auto_cls = type(automation)
@@ -47,6 +50,7 @@ class SlotRunner:
         self._on_output = on_output
         self._backend: Optional[Backend] = None
         self._provider_cache: Optional[tuple[str, str, ProxyProvider]] = None
+        self._browser_registry = browser_registry
 
     async def _ensure_backend(self) -> Backend:
         if self._backend is None:
@@ -77,6 +81,13 @@ class SlotRunner:
             except Exception:
                 log.exception("provider acquire failed; running without proxy")
 
+        # Resolve browser resource from registry if the automation declared
+        # a Browser need. Same instance is shared across all slots of this
+        # automation in this worker — the pool is process-wide.
+        browser_resource = None
+        if self._browser_registry is not None and self._auto_cls.Browser is not None:
+            browser_resource = self._browser_registry.register(self._auto_cls.Browser)
+
         ctx = Context(
             automation_type=self._auto.automation_type,
             worker_id=self._worker_id,
@@ -91,6 +102,7 @@ class SlotRunner:
             _automation_cls=self._auto_cls,
             _proxy=proxy_handle,
             proxy_url=proxy_url,
+            browser=browser_resource,
         )
 
         if isinstance(self._auto, ContinuousAutomation):

{fleet_framework-0.2.0 → fleet_framework-0.3.0}/fleet_browser/__init__.py

@@ -1,8 +1,7 @@
 from fleet_browser.browser import BrowserConfig, ChromiumWorker
+from fleet_browser.display import ensure_virtual_display
 from fleet_browser.fingerprint import Fingerprint, FingerprintFactory
-from fleet_browser.humanizer import Humanizer
-from fleet_browser.pool import BrowserPool, slot
-from fleet_browser.proxy_extension import build_proxy_auth_extension, parse_proxy_url
+from fleet_browser.pool import BrowserPool, make_browser_pool, slot
 from fleet_browser.smart_router import DEFAULT_RULES, RouterStats, SmartRouter, SmartRule
 from fleet_browser.stealth import FingerprintStealth, NoOpStealth, Stealth
 
@@ -14,13 +13,12 @@ __all__ = [
     "Fingerprint",
     "FingerprintFactory",
     "FingerprintStealth",
-    "Humanizer",
     "NoOpStealth",
     "RouterStats",
     "SmartRouter",
     "SmartRule",
     "Stealth",
-    "build_proxy_auth_extension",
-    "parse_proxy_url",
+    "ensure_virtual_display",
+    "make_browser_pool",
     "slot",
 ]