patchtroy 0.4.0__py3-none-any.whl

patchtroy/models.py

@@ -0,0 +1,154 @@
+"""Data schemas and models for Patchtroy."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class PatchtroyConfig(BaseModel):
+    """Configuration options for Patchtroy crawler execution."""
+
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    headless: bool = Field(
+        default=True,
+        description="Run Chromium in headless mode."
+    )
+    browser_timeout_s: float = Field(
+        default=25.0,
+        ge=2.0,
+        le=120.0,
+        description="Maximum seconds to wait for page load before fallback or error."
+    )
+    wait_for: str | None = Field(
+        default=None,
+        description="Optional CSS selector to wait for before extracting page content."
+    )
+    wait_until: str = Field(
+        default="domcontentloaded",
+        description="Navigation lifecycle event: 'domcontentloaded', 'load', or 'networkidle'."
+    )
+    user_agent: str | None = Field(
+        default=None,
+        description="Custom User-Agent string. If omitted, uses realistic browser UA."
+    )
+    http_fallback: bool = Field(
+        default=True,
+        description="Automatically fall back to direct HTTP request if browser crashes or times out."
+    )
+    extract_schema: bool = Field(
+        default=True,
+        description="Extract JSON-LD and Next.js __NEXT_DATA__ structured items."
+    )
+    extract_links: bool = Field(
+        default=True,
+        description="Extract all valid hyperlinks from the page."
+    )
+    viewport_width: int = Field(default=1280, ge=320, le=3840)
+    viewport_height: int = Field(default=800, ge=240, le=2160)
+    proxy: str | None = Field(
+        default=None,
+        description="Single proxy URL (e.g. 'http://user:pass@host:port')."
+    )
+    proxies: list[str] | str | None = Field(
+        default=None,
+        description="List of proxy URLs, comma-separated string, or path to proxy file for rotation."
+    )
+    proxy_strategy: str = Field(
+        default="round-robin",
+        description="Proxy rotation strategy: 'round-robin' or 'random'."
+    )
+    max_concurrency: int = Field(
+        default=5,
+        ge=1,
+        le=50,
+        description="Maximum concurrent browser contexts for batch scraping."
+    )
+    screenshot: bool = Field(
+        default=False,
+        description="Capture page screenshot."
+    )
+    full_page_screenshot: bool = Field(
+        default=False,
+        description="Capture full-page screenshot instead of viewport only."
+    )
+    screenshot_path: str | None = Field(
+        default=None,
+        description="Optional file path to automatically save screenshot to."
+    )
+    pdf: bool = Field(
+        default=False,
+        description="Generate page PDF (headless Chromium only)."
+    )
+    pdf_path: str | None = Field(
+        default=None,
+        description="Optional file path to automatically save PDF to."
+    )
+    custom_schema: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional declarative custom CSS selector schema for structured item harvesting."
+    )
+
+
+class LinkItem(BaseModel):
+    """Hyperlink extracted from rendered page."""
+    href: str
+    text: str = ""
+
+
+class ScrapeResult(BaseModel):
+    """Result emitted from a Patchtroy scrape operation."""
+
+    url: str
+    status_code: int = 200
+    title: str = ""
+    markdown: str = ""
+    html: str = ""
+    structured_data: list[dict[str, Any]] = Field(default_factory=list)
+    links: list[LinkItem] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    screenshot_bytes: bytes | None = None
+    pdf_bytes: bytes | None = None
+    success: bool = True
+    error: str | None = None
+    engine_used: str = "patchright"
+    elapsed_s: float = 0.0
+
+    @property
+    def has_content(self) -> bool:
+        return bool(self.markdown.strip() or self.structured_data)
+
+    def save_screenshot(self, filepath: str | Path) -> Path:
+        """Save screenshot bytes to disk."""
+        if not self.screenshot_bytes:
+            raise ValueError(f"No screenshot bytes available for {self.url}")
+        dest = Path(filepath)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(self.screenshot_bytes)
+        return dest
+
+    def save_pdf(self, filepath: str | Path) -> Path:
+        """Save PDF bytes to disk."""
+        if not self.pdf_bytes:
+            raise ValueError(f"No PDF bytes available for {self.url}")
+        dest = Path(filepath)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(self.pdf_bytes)
+        return dest
+
+    def chunk(
+        self,
+        max_tokens: int = 2048,
+        overlap_tokens: int = 100,
+    ) -> list[Any]:
+        """Split extracted Markdown content into LLM-ready TextChunk records."""
+        from patchtroy.chunker import chunk_markdown
+        return chunk_markdown(
+            self.markdown,
+            max_tokens=max_tokens,
+            overlap_tokens=overlap_tokens,
+            metadata={"url": self.url, "title": self.title},
+        )

patchtroy/pool.py

@@ -0,0 +1,135 @@
+"""Browser context pool for high-throughput concurrent scraping in Patchtroy."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+from patchtroy.models import PatchtroyConfig
+from patchtroy.proxy import ProxyManager
+
+logger = logging.getLogger("patchtroy.pool")
+
+
+class BrowserContextPool:
+    """Manages a single browser process and leases isolated browser contexts concurrently."""
+
+    def __init__(
+        self,
+        config: PatchtroyConfig,
+        proxy_manager: ProxyManager | None = None,
+    ) -> None:
+        self.config = config
+        self.proxy_manager = proxy_manager
+        self.max_concurrency = config.max_concurrency
+        self._semaphore: asyncio.Semaphore | None = None
+        self._playwright: Any = None
+        self._browser: Any = None
+        self._active_contexts: int = 0
+        self._lock = asyncio.Lock()
+
+    @property
+    def is_running(self) -> bool:
+        """Check if underlying browser is launched and active."""
+        return self._browser is not None
+
+    async def start(self) -> None:
+        """Launch the shared browser instance."""
+        if self._browser is not None:
+            return
+
+        async with self._lock:
+            if self._browser is not None:
+                return
+
+            if self._semaphore is None:
+                self._semaphore = asyncio.Semaphore(self.max_concurrency)
+
+            try:
+                try:
+                    from patchright.async_api import async_playwright
+                except ImportError:
+                    from playwright.async_api import async_playwright
+
+                self._playwright = await async_playwright().start()
+                launch_args = [
+                    "--disable-blink-features=AutomationControlled",
+                    "--no-sandbox",
+                    "--disable-dev-shm-usage",
+                ]
+                self._browser = await self._playwright.chromium.launch(
+                    headless=self.config.headless,
+                    args=launch_args,
+                )
+                logger.info(
+                    "BrowserContextPool started with concurrency limit %d",
+                    self.max_concurrency,
+                )
+            except Exception as exc:
+                logger.warning("Failed to initialize BrowserContextPool browser: %s", exc)
+                self._browser = None
+
+    async def close(self) -> None:
+        """Terminate all pooled contexts and close the browser."""
+        async with self._lock:
+            if self._browser:
+                try:
+                    await self._browser.close()
+                except Exception:
+                    pass
+                self._browser = None
+
+            if self._playwright:
+                try:
+                    await self._playwright.stop()
+                except Exception:
+                    pass
+                self._playwright = None
+
+    @asynccontextmanager
+    async def acquire_context(
+        self,
+        user_agent: str,
+        proxy: str | None = None,
+    ) -> AsyncIterator[tuple[Any, str | None]]:
+        """Acquire a managed browser context bounded by the concurrency semaphore."""
+        if self._browser is None:
+            await self.start()
+        if self._browser is None:
+            raise RuntimeError("BrowserContextPool has no available browser instance.")
+
+        if self._semaphore is None:
+            self._semaphore = asyncio.Semaphore(self.max_concurrency)
+
+        async with self._semaphore:
+            chosen_proxy = proxy
+            if not chosen_proxy and self.proxy_manager and self.proxy_manager.has_proxies:
+                chosen_proxy = self.proxy_manager.get_proxy()
+            elif not chosen_proxy and self.config.proxy:
+                chosen_proxy = self.config.proxy
+
+            context_kwargs: dict[str, Any] = {
+                "user_agent": user_agent,
+                "viewport": {
+                    "width": self.config.viewport_width,
+                    "height": self.config.viewport_height,
+                },
+                "ignore_https_errors": True,
+                "locale": "en-US",
+            }
+            if chosen_proxy:
+                context_kwargs["proxy"] = {"server": chosen_proxy}
+
+            context = await self._browser.new_context(**context_kwargs)
+            self._active_contexts += 1
+            try:
+                yield context, chosen_proxy
+            finally:
+                self._active_contexts -= 1
+                try:
+                    await context.close()
+                except Exception:
+                    pass

patchtroy/proxy.py

@@ -0,0 +1,140 @@
+"""Proxy rotation manager for Patchtroy."""
+
+from __future__ import annotations
+
+import logging
+import random
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+logger = logging.getLogger("patchtroy.proxy")
+
+ProxyStrategy = Literal["round-robin", "random"]
+
+
+@dataclass
+class ProxyItem:
+    """Represents a single proxy with health and failure tracking."""
+
+    server: str
+    failures: int = 0
+    quarantined_until: float = 0.0
+
+    @property
+    def is_healthy(self) -> bool:
+        """Return True if proxy is not quarantined."""
+        return time.monotonic() >= self.quarantined_until
+
+    def mark_failure(self, max_failures: int = 3, quarantine_seconds: float = 60.0) -> None:
+        """Increment failure counter and quarantine if threshold is reached."""
+        self.failures += 1
+        if self.failures >= max_failures:
+            self.quarantined_until = time.monotonic() + quarantine_seconds
+            logger.warning(
+                "Proxy %s quarantined for %.1fs after %d failures",
+                self.server,
+                quarantine_seconds,
+                self.failures,
+            )
+
+    def mark_success(self) -> None:
+        """Reset failures counter upon a successful request."""
+        self.failures = 0
+        self.quarantined_until = 0.0
+
+
+class ProxyManager:
+    """Manages a pool of proxies with rotation strategies and fault recovery."""
+
+    def __init__(
+        self,
+        proxies: list[str] | str | Path | None = None,
+        strategy: ProxyStrategy = "round-robin",
+        max_failures: int = 3,
+        quarantine_seconds: float = 60.0,
+    ) -> None:
+        self.strategy: ProxyStrategy = strategy
+        self.max_failures = max_failures
+        self.quarantine_seconds = quarantine_seconds
+        self._index: int = 0
+        self._items: list[ProxyItem] = []
+
+        if proxies:
+            self.load_proxies(proxies)
+
+    def load_proxies(self, proxies: list[str] | str | Path) -> None:
+        """Load proxies from a list of URLs, a comma-separated string, or a file path."""
+        proxy_list: list[str] = []
+
+        if isinstance(proxies, (str, Path)):
+            path = Path(proxies)
+            if path.is_file():
+                content = path.read_text(encoding="utf-8")
+                proxy_list = [
+                    line.strip()
+                    for line in content.splitlines()
+                    if line.strip() and not line.strip().startswith("#")
+                ]
+            elif isinstance(proxies, str):
+                proxy_list = [p.strip() for p in proxies.split(",") if p.strip()]
+        elif isinstance(proxies, list):
+            proxy_list = [p.strip() for p in proxies if p and p.strip()]
+
+        # Deduplicate while preserving order
+        seen = set()
+        unique = []
+        for p in proxy_list:
+            # Ensure protocol prefix exists if not provided
+            normalized = p if "://" in p else f"http://{p}"
+            if normalized not in seen:
+                seen.add(normalized)
+                unique.append(normalized)
+
+        self._items = [ProxyItem(server=p) for p in unique]
+        self._index = 0
+        logger.info("Loaded %d proxies into ProxyManager", len(self._items))
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    @property
+    def has_proxies(self) -> bool:
+        """Return True if at least one proxy is loaded."""
+        return len(self._items) > 0
+
+    def get_proxy(self) -> str | None:
+        """Select next healthy proxy based on configured strategy."""
+        if not self._items:
+            return None
+
+        healthy = [item for item in self._items if item.is_healthy]
+        # If all proxies are currently quarantined, use least-recently quarantined as fallback
+        pool = healthy if healthy else self._items
+
+        if self.strategy == "random":
+            chosen = random.choice(pool)
+            return chosen.server
+
+        # Round-robin selection
+        chosen = pool[self._index % len(pool)]
+        self._index = (self._index + 1) % len(pool)
+        return chosen.server
+
+    def report_failure(self, proxy_server: str) -> None:
+        """Record a failure for the specified proxy server."""
+        for item in self._items:
+            if item.server == proxy_server:
+                item.mark_failure(
+                    max_failures=self.max_failures,
+                    quarantine_seconds=self.quarantine_seconds,
+                )
+                break
+
+    def report_success(self, proxy_server: str) -> None:
+        """Record a success for the specified proxy server."""
+        for item in self._items:
+            if item.server == proxy_server:
+                item.mark_success()
+                break

patchtroy/server.py

@@ -0,0 +1,170 @@
+"""FastAPI REST API microservice for Patchtroy."""
+
+from __future__ import annotations
+
+import base64
+import logging
+from contextlib import asynccontextmanager
+from typing import Any, AsyncIterator
+
+from pydantic import BaseModel, Field
+
+from patchtroy.crawler import AsyncPatchtroy
+from patchtroy.models import PatchtroyConfig, ScrapeResult
+
+logger = logging.getLogger("patchtroy.server")
+
+try:
+    from fastapi import FastAPI
+    from fastapi.responses import JSONResponse
+except ImportError as exc:
+    raise ImportError(
+        "FastAPI is required to run the Patchtroy REST microservice. "
+        "Install it via: pip install 'patchtroy[server]'"
+    ) from exc
+
+
+class ScrapeRequestBody(BaseModel):
+    """Payload for /scrape endpoint."""
+
+    url: str = Field(description="Target URL to scrape")
+    wait_for: str | None = Field(default=None, description="CSS selector to wait for")
+    wait_until: str = Field(
+        default="domcontentloaded",
+        description="Navigation lifecycle: domcontentloaded, load, networkidle",
+    )
+    timeout: float = Field(default=25.0, description="Navigation timeout in seconds")
+    screenshot: bool = Field(default=False, description="Whether to capture screenshot")
+    full_page: bool = Field(default=False, description="Full-page screenshot mode")
+    pdf: bool = Field(default=False, description="Whether to generate PDF (headless only)")
+    proxy: str | None = Field(default=None, description="Optional proxy URL for request")
+    custom_schema: dict[str, Any] | None = Field(
+        default=None, description="Custom declarative CSS schema"
+    )
+
+
+class BatchScrapeRequestBody(BaseModel):
+    """Payload for /scrape/batch endpoint."""
+
+    urls: list[str] = Field(description="List of target URLs to scrape")
+    wait_for: str | None = Field(default=None, description="CSS selector to wait for")
+    timeout: float = Field(default=25.0, description="Navigation timeout in seconds")
+    concurrency: int = Field(default=5, ge=1, le=50, description="Concurrent browser contexts")
+    custom_schema: dict[str, Any] | None = Field(
+        default=None, description="Custom declarative CSS schema"
+    )
+
+
+def _serialize_scrape_result(res: ScrapeResult) -> dict[str, Any]:
+    """Serialize ScrapeResult into a JSON-friendly dict, encoding media bytes as base64."""
+    data = res.model_dump(exclude={"screenshot_bytes", "pdf_bytes"})
+    if res.screenshot_bytes:
+        data["screenshot_base64"] = base64.b64encode(res.screenshot_bytes).decode("ascii")
+    if res.pdf_bytes:
+        data["pdf_base64"] = base64.b64encode(res.pdf_bytes).decode("ascii")
+    return data
+
+
+def create_app() -> FastAPI:
+    """Factory creating configured FastAPI microservice instance."""
+    crawler_instance: AsyncPatchtroy | None = None
+
+    async def get_crawler() -> AsyncPatchtroy:
+        nonlocal crawler_instance
+        if crawler_instance is None:
+            crawler_instance = AsyncPatchtroy()
+            await crawler_instance.start()
+        return crawler_instance
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        nonlocal crawler_instance
+        logger.info("Initializing Patchtroy microservice browser engine...")
+        await get_crawler()
+        yield
+        logger.info("Terminating Patchtroy microservice browser engine...")
+        if crawler_instance:
+            await crawler_instance.close()
+            crawler_instance = None
+
+    app = FastAPI(
+        title="Patchtroy REST Microservice",
+        description="Undetected stealth web scraper & clean Markdown extractor for LLMs.",
+        version="0.4.0",
+        lifespan=lifespan,
+    )
+
+    @app.get("/health")
+    async def health_check() -> dict[str, Any]:
+        """Microservice health and status check."""
+        return {
+            "status": "healthy",
+            "version": "0.4.0",
+            "engine": "patchright",
+            "active": crawler_instance is not None and crawler_instance._pool.is_running,
+        }
+
+    @app.post("/scrape")
+    async def scrape_single(body: ScrapeRequestBody) -> JSONResponse:
+        """Scrape a single target web page into clean Markdown."""
+        crawler = await get_crawler()
+
+        config = PatchtroyConfig(
+            browser_timeout_s=body.timeout,
+            wait_for=body.wait_for,
+            wait_until=body.wait_until,
+            screenshot=body.screenshot,
+            full_page_screenshot=body.full_page,
+            pdf=body.pdf,
+            proxy=body.proxy,
+            custom_schema=body.custom_schema,
+        )
+        # Create lightweight client sharing the application's browser pool
+        client = AsyncPatchtroy(config)
+        client._pool = crawler._pool
+
+        result = await client.scrape(
+            body.url,
+            wait_for=body.wait_for,
+            custom_schema=body.custom_schema,
+        )
+
+        return JSONResponse(content=_serialize_scrape_result(result))
+
+    @app.post("/scrape/batch")
+    async def scrape_batch(body: BatchScrapeRequestBody) -> JSONResponse:
+        """Scrape multiple target web pages concurrently."""
+        crawler = await get_crawler()
+
+        config = PatchtroyConfig(
+            browser_timeout_s=body.timeout,
+            wait_for=body.wait_for,
+            max_concurrency=body.concurrency,
+            custom_schema=body.custom_schema,
+        )
+        client = AsyncPatchtroy(config)
+        client._pool = crawler._pool
+
+        results = await client.scrape_many(
+            body.urls,
+            wait_for=body.wait_for,
+            custom_schema=body.custom_schema,
+        )
+
+        serialized = [_serialize_scrape_result(r) for r in results]
+        return JSONResponse(content=serialized)
+
+    return app
+
+
+def run_server(host: str = "0.0.0.0", port: int = 4013, reload: bool = False) -> None:
+    """Launch the Uvicorn ASGI server."""
+    try:
+        import uvicorn
+    except ImportError as exc:
+        raise ImportError(
+            "Uvicorn is required to run the server. "
+            "Install it via: pip install 'patchtroy[server]'"
+        ) from exc
+
+    uvicorn.run("patchtroy.server:create_app", host=host, port=port, reload=reload, factory=True)

patchtroy/utils.py

@@ -0,0 +1,54 @@
+"""Utility functions, stealth injection scripts, and headers."""
+
+from __future__ import annotations
+
+import random
+from urllib.parse import urlparse
+
+DEFAULT_USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+]
+
+STEALTH_INJECTION_SCRIPT = """
+// Evade common automated browser detection signatures
+Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+window.chrome = {
+    runtime: {},
+    loadTimes: function() {},
+    csi: function() {},
+    app: {}
+};
+// Emulate realistic plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => [1, 2, 3, 4, 5],
+});
+// Emulate standard languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['en-US', 'en'],
+});
+// Emulate permissions query
+if (window.navigator && window.navigator.permissions) {
+    const originalQuery = window.navigator.permissions.query;
+    window.navigator.permissions.query = (parameters) => (
+        parameters.name === 'notifications' ?
+            Promise.resolve({ state: 'default' }) :
+            originalQuery(parameters)
+    );
+}
+"""
+
+
+def get_random_user_agent() -> str:
+    """Return a realistic modern desktop browser User-Agent."""
+    return random.choice(DEFAULT_USER_AGENTS)
+
+
+def is_valid_url(url: str) -> bool:
+    """Validate URL scheme is http or https."""
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False