patchtroy 0.4.0__py3-none-any.whl

patchtroy/crawler.py

@@ -0,0 +1,387 @@
+"""Core crawler implementations for Patchtroy."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from patchtroy.extractors import (
+    extract_links,
+    extract_markdown_and_metadata,
+    extract_structured_data,
+)
+from patchtroy.models import LinkItem, PatchtroyConfig, ScrapeResult
+from patchtroy.pool import BrowserContextPool
+from patchtroy.proxy import ProxyManager
+from patchtroy.utils import STEALTH_INJECTION_SCRIPT, get_random_user_agent, is_valid_url
+
+logger = logging.getLogger("patchtroy.crawler")
+
+
+class AsyncPatchtroy:
+    """Asynchronous stealth web crawler powered by Patchright, Trafilatura, and Context Pooling."""
+
+    def __init__(self, config: PatchtroyConfig | dict[str, Any] | None = None) -> None:
+        if isinstance(config, dict):
+            self.config = PatchtroyConfig(**config)
+        elif isinstance(config, PatchtroyConfig):
+            self.config = config
+        else:
+            self.config = PatchtroyConfig()
+
+        # Initialize ProxyManager if configured
+        self.proxy_manager: ProxyManager | None = None
+        if self.config.proxies:
+            self.proxy_manager = ProxyManager(
+                proxies=self.config.proxies,
+                strategy=self.config.proxy_strategy,
+            )
+
+        # Initialize BrowserContextPool
+        self._pool = BrowserContextPool(self.config, proxy_manager=self.proxy_manager)
+
+    @property
+    def _browser(self) -> Any:
+        """Compatibility accessor for underlying browser."""
+        return self._pool._browser
+
+    @property
+    def _playwright(self) -> Any:
+        """Compatibility accessor for underlying playwright."""
+        return self._pool._playwright
+
+    async def __aenter__(self) -> AsyncPatchtroy:
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+        await self.close()
+
+    async def start(self) -> None:
+        """Launch underlying stealth Patchright Chromium browser process via pool."""
+        await self._pool.start()
+
+    async def close(self) -> None:
+        """Gracefully terminate browser pool and all contexts."""
+        await self._pool.close()
+
+    async def scrape(
+        self,
+        url: str,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> ScrapeResult:
+        """Scrape a target URL using stealth browser automation, with optional HTTP fallback."""
+        if not is_valid_url(url):
+            return ScrapeResult(
+                url=url,
+                status_code=400,
+                success=False,
+                error=f"Invalid HTTP/HTTPS URL: {url}",
+            )
+
+        start_time = time.perf_counter()
+        wait_for_selector = wait_for or self.config.wait_for
+        ua = self.config.user_agent or get_random_user_agent()
+
+        # 1. Attempt stealth browser navigation
+        try:
+            return await self._scrape_with_browser(url, ua, wait_for_selector, custom_schema, start_time)
+        except Exception as exc:
+            logger.info("Browser scrape encountered issue: %s", exc)
+            if self.config.http_fallback:
+                logger.info("Engaging HTTP fallback for %s", url)
+                return await self._scrape_with_http(url, ua, custom_schema, start_time, str(exc))
+            else:
+                elapsed = time.perf_counter() - start_time
+                return ScrapeResult(
+                    url=url,
+                    status_code=500,
+                    success=False,
+                    error=str(exc),
+                    elapsed_s=round(elapsed, 3),
+                )
+
+    async def _scrape_with_browser(
+        self,
+        url: str,
+        user_agent: str,
+        wait_for_selector: str | None,
+        custom_schema: dict[str, Any] | None,
+        start_time: float,
+    ) -> ScrapeResult:
+        """Execute in-browser rendering with stealth injection, context pooling, and media capture."""
+        timeout_ms = int(self.config.browser_timeout_s * 1000)
+
+        async with self._pool.acquire_context(user_agent=user_agent) as (context, used_proxy):
+            page = await context.new_page()
+            try:
+                # Apply stealth browser configuration
+                await page.add_init_script(STEALTH_INJECTION_SCRIPT)
+
+                resp = await page.goto(
+                    url,
+                    timeout=timeout_ms,
+                    wait_until=self.config.wait_until,
+                )
+
+                status_code = resp.status if resp else 200
+
+                if wait_for_selector:
+                    try:
+                        await page.wait_for_selector(wait_for_selector, timeout=7000)
+                    except Exception as wait_exc:
+                        logger.debug("wait_for selector '%s' timeout: %s", wait_for_selector, wait_exc)
+
+                # Media captures: Screenshot & PDF
+                screenshot_bytes: bytes | None = None
+                if self.config.screenshot or self.config.screenshot_path:
+                    screenshot_bytes = await page.screenshot(full_page=self.config.full_page_screenshot)
+                    if self.config.screenshot_path:
+                        dest = Path(self.config.screenshot_path)
+                        dest.parent.mkdir(parents=True, exist_ok=True)
+                        dest.write_bytes(screenshot_bytes)
+
+                pdf_bytes: bytes | None = None
+                if self.config.pdf or self.config.pdf_path:
+                    try:
+                        pdf_bytes = await page.pdf()
+                        if self.config.pdf_path:
+                            dest = Path(self.config.pdf_path)
+                            dest.parent.mkdir(parents=True, exist_ok=True)
+                            dest.write_bytes(pdf_bytes)
+                    except Exception as pdf_exc:
+                        logger.warning("PDF export failed (requires headless Chromium): %s", pdf_exc)
+
+                html = await page.content()
+                final_url = page.url or url
+
+                # Record proxy success
+                if used_proxy and self.proxy_manager:
+                    self.proxy_manager.report_success(used_proxy)
+
+                # Content extractions
+                markdown, title, metadata = extract_markdown_and_metadata(html, base_url=final_url)
+                structured_data = []
+                if self.config.extract_schema:
+                    schema_to_use = custom_schema or self.config.custom_schema
+                    structured_data = extract_structured_data(html, source_url=final_url, custom_schema=schema_to_use)
+
+                links = []
+                if self.config.extract_links:
+                    raw_links = extract_links(html, base_url=final_url)
+                    links = [LinkItem(**link_dict) for link_dict in raw_links]
+
+                elapsed = time.perf_counter() - start_time
+                return ScrapeResult(
+                    url=final_url,
+                    status_code=status_code,
+                    title=title,
+                    markdown=markdown,
+                    html=html,
+                    structured_data=structured_data,
+                    links=links,
+                    metadata=metadata,
+                    screenshot_bytes=screenshot_bytes,
+                    pdf_bytes=pdf_bytes,
+                    success=True,
+                    engine_used="patchright",
+                    elapsed_s=round(elapsed, 3),
+                )
+            except Exception:
+                if used_proxy and self.proxy_manager:
+                    self.proxy_manager.report_failure(used_proxy)
+                raise
+            finally:
+                await page.close()
+
+    async def _scrape_with_http(
+        self,
+        url: str,
+        user_agent: str,
+        custom_schema: dict[str, Any] | None,
+        start_time: float,
+        browser_error: str,
+    ) -> ScrapeResult:
+        """Fast direct HTTP fallback when browser engine is unavailable or timed out."""
+        headers = {"User-Agent": user_agent, "Accept-Language": "en-US,en;q=0.9"}
+        try:
+            async with httpx.AsyncClient(
+                follow_redirects=True,
+                timeout=self.config.browser_timeout_s,
+                verify=False,
+            ) as client:
+                resp = await client.get(url, headers=headers)
+                html = resp.text
+                final_url = str(resp.url)
+                status_code = resp.status_code
+
+            markdown, title, metadata = extract_markdown_and_metadata(html, base_url=final_url)
+            structured_data = []
+            if self.config.extract_schema:
+                schema_to_use = custom_schema or self.config.custom_schema
+                structured_data = extract_structured_data(html, source_url=final_url, custom_schema=schema_to_use)
+
+            links = []
+            if self.config.extract_links:
+                raw_links = extract_links(html, base_url=final_url)
+                links = [LinkItem(**link_dict) for link_dict in raw_links]
+
+            elapsed = time.perf_counter() - start_time
+            return ScrapeResult(
+                url=final_url,
+                status_code=status_code,
+                title=title,
+                markdown=markdown,
+                html=html,
+                structured_data=structured_data,
+                links=links,
+                metadata=metadata,
+                success=resp.is_success,
+                error=f"HTTP fallback (browser failed: {browser_error})",
+                engine_used="http_fallback",
+                elapsed_s=round(elapsed, 3),
+            )
+        except Exception as http_exc:
+            elapsed = time.perf_counter() - start_time
+            return ScrapeResult(
+                url=url,
+                status_code=500,
+                success=False,
+                error=f"Both browser and HTTP fallback failed. Browser: {browser_error} | HTTP: {http_exc}",
+                engine_used="failed",
+                elapsed_s=round(elapsed, 3),
+            )
+
+    async def scrape_many(
+        self,
+        urls: list[str],
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> list[ScrapeResult]:
+        """Scrape multiple URLs concurrently bounded by the BrowserContextPool."""
+        tasks = [
+            self.scrape(url, wait_for=wait_for, custom_schema=custom_schema)
+            for url in urls
+        ]
+        return await asyncio.gather(*tasks)
+
+    @classmethod
+    async def crawl(
+        cls,
+        url: str,
+        headless: bool = True,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+        screenshot: bool = False,
+        pdf: bool = False,
+    ) -> ScrapeResult:
+        """One-shot convenience coroutine for instant single-URL scraping."""
+        config = PatchtroyConfig(
+            headless=headless,
+            wait_for=wait_for,
+            screenshot=screenshot,
+            pdf=pdf,
+        )
+        async with cls(config) as client:
+            return await client.scrape(url, wait_for=wait_for, custom_schema=custom_schema)
+
+    @classmethod
+    async def crawl_many(
+        cls,
+        urls: list[str],
+        headless: bool = True,
+        max_concurrency: int = 5,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> list[ScrapeResult]:
+        """One-shot convenience coroutine for instant concurrent batch scraping."""
+        config = PatchtroyConfig(
+            headless=headless,
+            max_concurrency=max_concurrency,
+            wait_for=wait_for,
+            custom_schema=custom_schema,
+        )
+        async with cls(config) as client:
+            return await client.scrape_many(urls, wait_for=wait_for, custom_schema=custom_schema)
+
+
+class Patchtroy:
+    """Synchronous wrapper for Patchtroy crawler execution."""
+
+    def __init__(self, config: PatchtroyConfig | dict[str, Any] | None = None) -> None:
+        self._async_crawler = AsyncPatchtroy(config)
+
+    def scrape(
+        self,
+        url: str,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> ScrapeResult:
+        """Execute scrape synchronously in an event loop."""
+        if sys.platform == "win32":
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+        return asyncio.run(self._async_crawler.scrape(url, wait_for=wait_for, custom_schema=custom_schema))
+
+    def scrape_many(
+        self,
+        urls: list[str],
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> list[ScrapeResult]:
+        """Execute concurrent batch scraping synchronously."""
+        if sys.platform == "win32":
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+        return asyncio.run(self._async_crawler.scrape_many(urls, wait_for=wait_for, custom_schema=custom_schema))
+
+    @classmethod
+    def crawl(
+        cls,
+        url: str,
+        headless: bool = True,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+        screenshot: bool = False,
+        pdf: bool = False,
+    ) -> ScrapeResult:
+        """Synchronous one-shot convenience function for scraping."""
+        if sys.platform == "win32":
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+        return asyncio.run(
+            AsyncPatchtroy.crawl(
+                url,
+                headless=headless,
+                wait_for=wait_for,
+                custom_schema=custom_schema,
+                screenshot=screenshot,
+                pdf=pdf,
+            )
+        )
+
+    @classmethod
+    def crawl_many(
+        cls,
+        urls: list[str],
+        headless: bool = True,
+        max_concurrency: int = 5,
+        wait_for: str | None = None,
+        custom_schema: dict[str, Any] | None = None,
+    ) -> list[ScrapeResult]:
+        """Synchronous one-shot convenience function for concurrent batch crawling."""
+        if sys.platform == "win32":
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+        return asyncio.run(
+            AsyncPatchtroy.crawl_many(
+                urls,
+                headless=headless,
+                max_concurrency=max_concurrency,
+                wait_for=wait_for,
+                custom_schema=custom_schema,
+            )
+        )

patchtroy/extractors.py

@@ -0,0 +1,207 @@
+"""Content and structured data extraction engine for Patchtroy."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+from urllib.parse import urljoin
+
+import trafilatura
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger('patchtroy.extractors')
+
+
+def extract_markdown_and_metadata(
+    html: str,
+    base_url: str = ''
+) -> tuple[str, str, dict[str, Any]]:
+    if not html or not html.strip():
+        return '', '', {}
+
+    title = ''
+    metadata: dict[str, Any] = {}
+
+    try:
+        soup = BeautifulSoup(html, 'html.parser')
+        title_el = soup.find('title')
+        if title_el and title_el.string:
+            title = title_el.string.strip()
+    except Exception:
+        pass
+
+    try:
+        meta = trafilatura.extract_metadata(html, default_url=base_url)
+        if meta:
+            metadata = {
+                'title': meta.title,
+                'author': meta.author,
+                'url': meta.url or base_url,
+                'hostname': meta.hostname,
+                'description': meta.description,
+                'sitename': meta.sitename,
+                'date': meta.date,
+                'categories': meta.categories,
+                'tags': meta.tags,
+            }
+            if meta.title:
+                title = meta.title
+    except Exception as exc:
+        logger.debug('Trafilatura metadata extraction error: %s', exc)
+
+    markdown = ''
+    try:
+        extracted = trafilatura.extract(
+            html,
+            url=base_url,
+            output_format='markdown',
+            include_links=True,
+            include_images=True,
+            include_formatting=True,
+            include_tables=True,
+            favor_recall=True,
+        )
+        if extracted:
+            markdown = extracted.strip()
+    except Exception as exc:
+        logger.debug('Trafilatura content extraction error: %s', exc)
+
+    if not markdown:
+        try:
+            soup = BeautifulSoup(html, 'html.parser')
+            for tag in soup(['script', 'style', 'noscript', 'svg']):
+                tag.decompose()
+            body = soup.find('body') or soup
+            lines = [line.strip() for line in body.get_text(separator=' ').splitlines()]
+            markdown = chr(10).join(line for line in lines if line)
+        except Exception:
+            markdown = ''
+
+    return markdown, title, metadata
+
+
+def extract_structured_data(
+    html: str,
+    source_url: str = '',
+    custom_schema: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    if not html or not html.strip():
+        return []
+
+    items: list[dict[str, Any]] = []
+
+    try:
+        soup = BeautifulSoup(html, 'html.parser')
+    except Exception as exc:
+        logger.warning('BeautifulSoup parsing failed: %s', exc)
+        return []
+
+    if custom_schema and isinstance(custom_schema, dict) and custom_schema.get('fields'):
+        custom_items = _extract_custom_schema(soup, source_url, custom_schema)
+        if custom_items:
+            return custom_items
+
+    next_data_el = soup.find('script', id='__NEXT_DATA__')
+    if next_data_el and next_data_el.string:
+        try:
+            nd = json.loads(next_data_el.string)
+            props = nd.get('props', {}).get('pageProps', {})
+            raw_listings = (
+                props.get('listings')
+                or props.get('products')
+                or props.get('items')
+                or props.get('data')
+            )
+            if isinstance(raw_listings, list) and raw_listings:
+                for item in raw_listings:
+                    if isinstance(item, dict):
+                        item['_source_url'] = source_url
+                        items.append(item)
+                if items:
+                    return items
+        except Exception:
+            pass
+
+    ld_items = _extract_json_ld(soup, source_url)
+    if ld_items:
+        items.extend(ld_items)
+
+    return items
+
+
+def extract_links(html: str, base_url: str = '') -> list[dict[str, str]]:
+    if not html:
+        return []
+
+    links: list[dict[str, str]] = []
+    seen: set[str] = set()
+
+    try:
+        soup = BeautifulSoup(html, 'html.parser')
+        for a in soup.find_all('a', href=True):
+            href = a['href'].strip()
+            if not href or href.startswith(('javascript:', 'mailto:', 'tel:', '#')):
+                continue
+            full_url = urljoin(base_url, href) if base_url else href
+            if full_url not in seen:
+                seen.add(full_url)
+                text = a.get_text(strip=True)
+                links.append({'href': full_url, 'text': text})
+    except Exception:
+        pass
+
+    return links
+
+
+def _extract_json_ld(soup: BeautifulSoup, source_url: str) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+
+    for tag in soup.find_all('script', type='application/ld+json'):
+        if not tag.string:
+            continue
+        try:
+            data = json.loads(tag.string)
+            elements = data if isinstance(data, list) else [data]
+            for el in elements:
+                if not isinstance(el, dict):
+                    continue
+                if '@graph' in el and isinstance(el['@graph'], list):
+                    for sub in el['@graph']:
+                        if isinstance(sub, dict):
+                            sub['_source_url'] = source_url
+                            items.append(sub)
+                else:
+                    el['_source_url'] = source_url
+                    items.append(el)
+        except Exception:
+            continue
+
+    return items
+
+
+def _extract_custom_schema(
+    soup: BeautifulSoup,
+    source_url: str,
+    schema: dict[str, Any]
+) -> list[dict[str, Any]]:
+    item_selector = schema.get('item_selector')
+    fields = schema.get('fields', {})
+    results: list[dict[str, Any]] = []
+
+    if item_selector:
+        containers = soup.select(item_selector)
+        for container in containers:
+            record: dict[str, Any] = {'_source_url': source_url}
+            for field_name, selector in fields.items():
+                el = container.select_one(selector)
+                record[field_name] = el.get_text(strip=True) if el else None
+            results.append(record)
+    else:
+        record = {'_source_url': source_url}
+        for field_name, selector in fields.items():
+            el = soup.select_one(selector)
+            record[field_name] = el.get_text(strip=True) if el else None
+        results.append(record)
+
+    return results