scrapeflow-py 0.1.0__py3-none-any.whl

examples/__init__.py

@@ -0,0 +1,2 @@
+"""Examples for ScrapeFlow usage."""
+

examples/advanced_example.py

@@ -0,0 +1,80 @@
+"""Advanced example showing anti-detection, rate limiting, and structured extraction."""
+
+import asyncio
+from scrapeflow import ScrapeFlow
+from scrapeflow.config import (
+    ScrapeFlowConfig,
+    AntiDetectionConfig,
+    RateLimitConfig,
+    RetryConfig,
+)
+from scrapeflow.extractors import StructuredExtractor
+
+
+async def main():
+    """Advanced scraping with all features enabled."""
+    # Configure anti-detection
+    anti_detection = AntiDetectionConfig(
+        rotate_user_agents=True,
+        stealth_mode=True,
+        viewport_width=1920,
+        viewport_height=1080,
+    )
+
+    # Configure rate limiting
+    rate_limit = RateLimitConfig(
+        requests_per_second=1.0,
+        burst_size=3,
+    )
+
+    # Configure retry behavior
+    retry = RetryConfig(
+        max_retries=5,
+        initial_delay=1.0,
+        max_delay=30.0,
+        exponential_base=2.0,
+    )
+
+    # Create config
+    config = ScrapeFlowConfig(
+        anti_detection=anti_detection,
+        rate_limit=rate_limit,
+        retry=retry,
+        log_level="INFO",
+        debug=False,
+    )
+
+    async with ScrapeFlow(config) as scraper:
+        # Navigate to page
+        await scraper.navigate("https://quotes.toscrape.com/")
+
+        # Use structured extractor
+        schema = {
+            "quotes": {
+                "items": ".quote",
+                "schema": {
+                    "text": ".text",
+                    "author": ".author",
+                    "tags": ".tag",
+                },
+            }
+        }
+
+        extractor = StructuredExtractor(schema)
+        data = await extractor.extract(scraper.page)
+
+        print("Extracted quotes:")
+        for quote in data.get("quotes", []):
+            print(f"  - {quote}")
+
+        # Get metrics
+        metrics = scraper.get_metrics()
+        print(f"\nScraping completed:")
+        print(f"  Success rate: {metrics.get_success_rate():.2f}%")
+        print(f"  Total requests: {metrics.total_requests}")
+        print(f"  Retries: {metrics.retry_count}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/basic_usage.py

@@ -0,0 +1,34 @@
+"""Basic usage example of ScrapeFlow."""
+
+import asyncio
+from scrapeflow import ScrapeFlow
+from scrapeflow.config import ScrapeFlowConfig
+
+
+async def main():
+    """Basic scraping example."""
+    # Create a simple configuration
+    config = ScrapeFlowConfig()
+    config.browser.headless = False  # Set to True for headless mode
+    config.retry.max_retries = 3
+
+    # Use ScrapeFlow as a context manager
+    async with ScrapeFlow(config) as scraper:
+        # Navigate to a page
+        await scraper.navigate("https://example.com")
+
+        # Take a screenshot
+        await scraper.screenshot("example.png")
+
+        # Extract some data
+        title = await scraper.page.title()
+        print(f"Page title: {title}")
+
+        # Get metrics
+        metrics = scraper.get_metrics()
+        print(f"Success rate: {metrics.get_success_rate():.2f}%")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/workflow_example.py

@@ -0,0 +1,86 @@
+"""Workflow example showing how to use ScrapeFlow workflows."""
+
+import asyncio
+from scrapeflow import ScrapeFlow, Workflow
+from scrapeflow.config import ScrapeFlowConfig
+from scrapeflow.extractors import Extractor
+
+
+async def scrape_product_page(page, context):
+    """Extract product information from a page."""
+    # Wait for product details to load
+    await page.wait_for_selector(".product-title", timeout=10000)
+
+    # Extract product data
+    product_data = {
+        "title": await Extractor.extract_text(page, ".product-title"),
+        "price": await Extractor.extract_text(page, ".price"),
+        "description": await Extractor.extract_text(page, ".description"),
+        "images": await Extractor.extract_images(page, ".product-image"),
+    }
+
+    context["product_data"] = product_data
+    return product_data
+
+
+async def save_data(data, context):
+    """Save extracted data (mock function)."""
+    print(f"Saving data: {data}")
+    # In real usage, you would save to database, file, etc.
+    return True
+
+
+async def handle_error(error, context):
+    """Handle errors during scraping."""
+    print(f"Error occurred: {error}")
+    # You could log to file, send notification, etc.
+
+
+async def main():
+    """Workflow example."""
+    config = ScrapeFlowConfig()
+    config.retry.max_retries = 3
+    config.rate_limit.requests_per_second = 2.0
+
+    async with ScrapeFlow(config) as scraper:
+        # Create a workflow
+        workflow = Workflow(name="product_scraper")
+
+        # Add steps
+        async def navigate_to_product(page, context):
+            await scraper.navigate("https://example.com/products/1")
+
+        workflow.add_step(
+            name="navigate",
+            func=navigate_to_product,
+        )
+
+        workflow.add_step(
+            name="extract_data",
+            func=scrape_product_page,
+            retryable=True,
+            required=True,
+            on_success=save_data,
+            on_error=handle_error,
+        )
+
+        # Execute workflow
+        result = await scraper.run_workflow(workflow)
+
+        if result.success:
+            print("Workflow completed successfully!")
+            print(f"Extracted data: {result.final_data}")
+        else:
+            print(f"Workflow failed: {result.error}")
+
+        # Print metrics
+        metrics = scraper.get_metrics()
+        print(f"\nMetrics:")
+        print(f"  Total requests: {metrics.total_requests}")
+        print(f"  Success rate: {metrics.get_success_rate():.2f}%")
+        print(f"  Average response time: {metrics.average_response_time:.2f}s")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

scrapeflow/__init__.py

@@ -0,0 +1,30 @@
+"""
+ScrapeFlow - An opinionated scraping workflow engine built on Playwright.
+
+ScrapeFlow provides a robust, production-ready framework for web scraping
+with built-in error handling, retry logic, anti-detection, and workflow management.
+"""
+
+from scrapeflow.engine import ScrapeFlow
+from scrapeflow.workflow import Workflow, Step
+from scrapeflow.extractors import Extractor, StructuredExtractor
+from scrapeflow.exceptions import (
+    ScrapeFlowError,
+    ScrapeFlowRetryError,
+    ScrapeFlowTimeoutError,
+    ScrapeFlowBlockedError,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "ScrapeFlow",
+    "Workflow",
+    "Step",
+    "Extractor",
+    "StructuredExtractor",
+    "ScrapeFlowError",
+    "ScrapeFlowRetryError",
+    "ScrapeFlowTimeoutError",
+    "ScrapeFlowBlockedError",
+]
+

scrapeflow/anti_detection.py

@@ -0,0 +1,140 @@
+"""Anti-detection utilities for web scraping."""
+
+import random
+from typing import List, Optional, Dict
+from scrapeflow.config import AntiDetectionConfig
+
+
+class UserAgentRotator:
+    """Manages user agent rotation."""
+
+    DEFAULT_USER_AGENTS = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    ]
+
+    def __init__(self, user_agents: Optional[List[str]] = None):
+        self.user_agents = user_agents or self.DEFAULT_USER_AGENTS
+
+    def get_random(self) -> str:
+        """Get a random user agent."""
+        return random.choice(self.user_agents)
+
+    def get(self, index: int = None) -> str:
+        """Get a user agent by index or random."""
+        if index is not None:
+            return self.user_agents[index % len(self.user_agents)]
+        return self.get_random()
+
+
+class ProxyRotator:
+    """Manages proxy rotation."""
+
+    def __init__(self, proxies: List[Dict[str, str]]):
+        self.proxies = proxies
+        self.current_index = 0
+
+    def get_next(self) -> Optional[Dict[str, str]]:
+        """Get the next proxy in rotation."""
+        if not self.proxies:
+            return None
+        proxy = self.proxies[self.current_index % len(self.proxies)]
+        self.current_index += 1
+        return proxy
+
+    def get_random(self) -> Optional[Dict[str, str]]:
+        """Get a random proxy."""
+        if not self.proxies:
+            return None
+        return random.choice(self.proxies)
+
+
+class StealthMode:
+    """Applies stealth techniques to avoid detection."""
+
+    @staticmethod
+    async def apply_stealth(page):
+        """Apply stealth techniques to a Playwright page."""
+        # Remove webdriver property
+        await page.add_init_script(
+            """
+            Object.defineProperty(navigator, 'webdriver', {
+                get: () => undefined
+            });
+            """
+        )
+
+        # Override permissions
+        await page.add_init_script(
+            """
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => (
+                parameters.name === 'notifications' ?
+                    Promise.resolve({ state: Notification.permission }) :
+                    originalQuery(parameters)
+            );
+            """
+        )
+
+        # Override plugins
+        await page.add_init_script(
+            """
+            Object.defineProperty(navigator, 'plugins', {
+                get: () => [1, 2, 3, 4, 5]
+            });
+            """
+        )
+
+        # Override languages
+        await page.add_init_script(
+            """
+            Object.defineProperty(navigator, 'languages', {
+                get: () => ['en-US', 'en']
+            });
+            """
+        )
+
+        # Mock chrome object
+        await page.add_init_script(
+            """
+            window.chrome = {
+                runtime: {}
+            };
+            """
+        )
+
+
+class AntiDetectionManager:
+    """Manages all anti-detection features."""
+
+    def __init__(self, config: AntiDetectionConfig):
+        self.config = config
+        self.user_agent_rotator = (
+            UserAgentRotator(config.user_agents) if config.rotate_user_agents else None
+        )
+        self.proxy_rotator = (
+            ProxyRotator(config.proxies) if config.rotate_proxies else None
+        )
+
+    def get_user_agent(self) -> Optional[str]:
+        """Get a user agent if rotation is enabled."""
+        if self.user_agent_rotator:
+            return self.user_agent_rotator.get_random()
+        return None
+
+    def get_proxy(self) -> Optional[Dict[str, str]]:
+        """Get a proxy if rotation is enabled."""
+        if self.proxy_rotator:
+            return self.proxy_rotator.get_next()
+        return None
+
+    async def apply_stealth(self, page):
+        """Apply stealth mode to a page if enabled."""
+        if self.config.stealth_mode:
+            await StealthMode.apply_stealth(page)
+

scrapeflow/config.py

@@ -0,0 +1,75 @@
+"""Configuration and settings for ScrapeFlow."""
+
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+from enum import Enum
+
+
+class BrowserType(str, Enum):
+    """Supported browser types."""
+
+    CHROMIUM = "chromium"
+    FIREFOX = "firefox"
+    WEBKIT = "webkit"
+
+
+@dataclass
+class RetryConfig:
+    """Configuration for retry behavior."""
+
+    max_retries: int = 3
+    initial_delay: float = 1.0
+    max_delay: float = 60.0
+    exponential_base: float = 2.0
+    jitter: bool = True
+    retryable_exceptions: tuple = (
+        Exception,
+    )  # Will be customized per use case
+
+
+@dataclass
+class RateLimitConfig:
+    """Configuration for rate limiting."""
+
+    requests_per_second: float = 1.0
+    requests_per_minute: Optional[float] = None
+    burst_size: int = 5
+
+
+@dataclass
+class AntiDetectionConfig:
+    """Configuration for anti-detection features."""
+
+    rotate_user_agents: bool = True
+    rotate_proxies: bool = False
+    stealth_mode: bool = True
+    viewport_width: int = 1920
+    viewport_height: int = 1080
+    user_agents: List[str] = field(default_factory=list)
+    proxies: List[Dict[str, str]] = field(default_factory=list)
+
+
+@dataclass
+class BrowserConfig:
+    """Configuration for browser settings."""
+
+    browser_type: BrowserType = BrowserType.CHROMIUM
+    headless: bool = True
+    timeout: int = 30000  # milliseconds
+    slow_mo: int = 0  # milliseconds
+    args: List[str] = field(default_factory=list)
+    proxy: Optional[Dict[str, str]] = None
+
+
+@dataclass
+class ScrapeFlowConfig:
+    """Main configuration for ScrapeFlow."""
+
+    browser: BrowserConfig = field(default_factory=BrowserConfig)
+    retry: RetryConfig = field(default_factory=RetryConfig)
+    rate_limit: RateLimitConfig = field(default_factory=RateLimitConfig)
+    anti_detection: AntiDetectionConfig = field(default_factory=AntiDetectionConfig)
+    log_level: str = "INFO"
+    log_file: Optional[str] = None
+    debug: bool = False
+