scrapeflow-py 0.1.2__tar.gz → 0.2.0__tar.gz

{scrapeflow_py-0.1.2/scrapeflow_py.egg-info → scrapeflow_py-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeflow-py
-Version: 0.1.2
+Version: 0.2.0
 Summary: An opinionated scraping workflow engine built on Playwright
 Home-page: https://github.com/irfanalidv/scrapeflow-py
 Author: Irfan Ali
@@ -20,6 +20,7 @@ License-File: LICENSE
 Requires-Dist: playwright>=1.40.0
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: tenacity>=8.2.0
+Requires-Dist: pydantic>=2.0.0
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -35,17 +36,7 @@ Dynamic: summary
 
 **An opinionated scraping workflow engine built on Playwright**
 
-[![GitHub](https://img.shields.io/github/license/irfanalidv/scrapeflow-py)](https://github.com/irfanalidv/scrapeflow-py/blob/main/LICENSE)
-
-[![PyPI](https://img.shields.io/pypi/v/scrapeflow-py)](https://pypi.org/project/scrapeflow-py/)
-
-[![PyPI Downloads](https://static.pepy.tech/personalized-badge/scrapeflow-py?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/scrapeflow-py)
-
-[![Python](https://img.shields.io/badge/python-3.8%2B-blue)](https://www.python.org/)
-
-[![Playwright](https://img.shields.io/badge/Playwright-1.40%2B-green)](https://playwright.dev/)
-
-[![Status](https://img.shields.io/badge/status-active-success)](https://github.com/irfanalidv/scrapeflow-py)
+[![GitHub](https://img.shields.io/github/license/irfanalidv/scrapeflow-py)](https://github.com/irfanalidv/scrapeflow-py/blob/main/LICENSE) [![PyPI](https://img.shields.io/pypi/v/scrapeflow-py)](https://pypi.org/project/scrapeflow-py/) [![PyPI Downloads](https://static.pepy.tech/personalized-badge/scrapeflow-py?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/scrapeflow-py) [![Python](https://img.shields.io/badge/python-3.8%2B-blue)](https://www.python.org/) [![Playwright](https://img.shields.io/badge/Playwright-1.40%2B-green)](https://playwright.dev/) [![Status](https://img.shields.io/badge/status-active-success)](https://github.com/irfanalidv/scrapeflow-py)
 
 ---
 
@@ -53,6 +44,12 @@ ScrapeFlow is a production-ready Python library that transforms Playwright into
 
 ## 🚀 Features
 
+- **📋 Specification-Driven Extraction**: Declarative Pydantic models define fields, types, and validation—decouple field definitions from page structure
+- **🤖 robots.txt Compliance**: Built-in robots.txt parsing and enforcement; ethical crawling by design
+- **⚖️ Ethical Crawling (GDPR/CCPA)**: Configurable data retention, anonymization, and consent options in the specification layer
+- **📦 Component Registry**: Shared, versioned selectors, pagination handlers, and login flows—platform thinking over one-off scrapers
+- **🔔 Monitoring & Alerting**: Alert callbacks on failure thresholds; rollback hooks for failed extraction runs
+- **🔌 MCP Extensibility**: Pluggable backends for Scrapy MCP Server, Playwright MCP, or LLM-based semantic extraction
 - **🔄 Intelligent Retry Logic**: Automatic retries with exponential backoff and jitter
 - **⚡ Rate Limiting**: Token bucket algorithm to respect server limits
 - **🕵️ Anti-Detection**: Stealth mode, user agent rotation, and proxy support
@@ -62,6 +59,10 @@ ScrapeFlow is a production-ready Python library that transforms Playwright into
 - **🔧 Error Handling**: Comprehensive error classification and recovery
 - **📝 Type Hints**: Full type support for better IDE experience
 
+<p align="center">
+  <img src="docs/scrapeflow-architecture.png" alt="ScrapeFlow Architecture" width="800">
+</p>
+
 ## 📦 Installation
 
 ```bash
@@ -414,6 +415,63 @@ config = ScrapeFlowConfig(
 )
 ```
 
+### Specification-Driven Extraction (Pydantic)
+
+**Use Case:** Declarative extraction with validation—fields, types, and rules in specs, not fragile XPaths.
+
+```python
+from pydantic import BaseModel
+from scrapeflow import ScrapeFlow, SpecificationExtractor
+from scrapeflow.specifications import FieldSpec, ItemSpec, ProductPriceSpec
+
+# Model for list of products
+class BookListing(BaseModel):
+    books: list[ProductPriceSpec]
+
+# Schema maps fields to selectors
+schema = {
+    "books": ItemSpec(
+        items_selector="article.product_pod",
+        fields={
+            "title": FieldSpec(selector="h3 a"),
+            "price": FieldSpec(selector=".price_color"),
+            "availability": FieldSpec(selector=".instock.availability", default=""),
+            "url": FieldSpec(selector="h3 a", type="attribute", attribute="href"),
+        },
+    )
+}
+
+async with ScrapeFlow() as scraper:
+    await scraper.navigate("https://books.toscrape.com/")
+    extractor = SpecificationExtractor(BookListing, schema=schema)
+    # Extract and validate in one step
+    data = await extractor.extract(scraper.page)
+    for book in data.books:
+        print(f"{book.title}: {book.price}")
+```
+
+### Ethical Crawling & robots.txt
+
+**Use Case:** GDPR/CCPA compliance and robots.txt respect built into the specification layer.
+
+```python
+from scrapeflow import ScrapeFlow
+from scrapeflow.config import ScrapeFlowConfig, EthicalCrawlingConfig
+
+config = ScrapeFlowConfig(
+    ethical_crawling=EthicalCrawlingConfig(
+        respect_robots_txt=True,      # Check robots.txt before each request
+        user_agent_for_robots="ScrapeFlow",
+        anonymize_ip=True,            # GDPR: minimize personal data
+        data_retention_days=30,       # Document retention policy
+    )
+)
+
+async with ScrapeFlow(config) as scraper:
+    # navigate() automatically checks robots.txt
+    await scraper.navigate("https://example.com/page")
+```
+
 ### Anti-Detection
 
 **Use Case:** Scraping protected e-commerce sites that block automated access.
@@ -839,16 +897,26 @@ ScrapeFlow is built with a modular architecture:
 ```
 scrapeflow/
 ├── engine.py          # Main ScrapeFlow engine
-├── workflow.py        # Workflow definition and execution
-├── config.py          # Configuration classes
+├── ports.py           # Protocols for dependency inversion
+├── browser_runtime.py # Playwright runtime adapter
+├── workflow.py        # Workflow definition entities
+├── workflow_executor.py # Workflow execution service
+├── config.py          # Configuration classes (incl. EthicalCrawlingConfig)
+├── specifications.py  # Pydantic specification-driven extraction
+├── schema_library.py  # Reusable schema definitions
+├── robots.py          # robots.txt parsing and enforcement
+├── registry.py        # Shared selector/component registry
+├── mcp_backend.py     # MCP integration extensibility
 ├── anti_detection.py  # Anti-detection utilities
 ├── rate_limiter.py    # Rate limiting implementation
 ├── retry.py           # Retry logic and error classification
-├── monitoring.py      # Metrics and logging
+├── monitoring.py      # Metrics, logging, alerting
 ├── extractors.py      # Data extraction utilities
 └── exceptions.py      # Custom exceptions
 ```
 
+For deeper design details, see [`ARCHITECTURE.md`](ARCHITECTURE.md).
+
 ## 🤝 Contributing
 
 Contributions are welcome! Please feel free to submit a Pull Request.

{scrapeflow_py-0.1.2 → scrapeflow_py-0.2.0}/README.md

@@ -2,17 +2,7 @@
 
 **An opinionated scraping workflow engine built on Playwright**
 
-[![GitHub](https://img.shields.io/github/license/irfanalidv/scrapeflow-py)](https://github.com/irfanalidv/scrapeflow-py/blob/main/LICENSE)
-
-[![PyPI](https://img.shields.io/pypi/v/scrapeflow-py)](https://pypi.org/project/scrapeflow-py/)
-
-[![PyPI Downloads](https://static.pepy.tech/personalized-badge/scrapeflow-py?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/scrapeflow-py)
-
-[![Python](https://img.shields.io/badge/python-3.8%2B-blue)](https://www.python.org/)
-
-[![Playwright](https://img.shields.io/badge/Playwright-1.40%2B-green)](https://playwright.dev/)
-
-[![Status](https://img.shields.io/badge/status-active-success)](https://github.com/irfanalidv/scrapeflow-py)
+[![GitHub](https://img.shields.io/github/license/irfanalidv/scrapeflow-py)](https://github.com/irfanalidv/scrapeflow-py/blob/main/LICENSE) [![PyPI](https://img.shields.io/pypi/v/scrapeflow-py)](https://pypi.org/project/scrapeflow-py/) [![PyPI Downloads](https://static.pepy.tech/personalized-badge/scrapeflow-py?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/scrapeflow-py) [![Python](https://img.shields.io/badge/python-3.8%2B-blue)](https://www.python.org/) [![Playwright](https://img.shields.io/badge/Playwright-1.40%2B-green)](https://playwright.dev/) [![Status](https://img.shields.io/badge/status-active-success)](https://github.com/irfanalidv/scrapeflow-py)
 
 ---
 
@@ -20,6 +10,12 @@ ScrapeFlow is a production-ready Python library that transforms Playwright into
 
 ## 🚀 Features
 
+- **📋 Specification-Driven Extraction**: Declarative Pydantic models define fields, types, and validation—decouple field definitions from page structure
+- **🤖 robots.txt Compliance**: Built-in robots.txt parsing and enforcement; ethical crawling by design
+- **⚖️ Ethical Crawling (GDPR/CCPA)**: Configurable data retention, anonymization, and consent options in the specification layer
+- **📦 Component Registry**: Shared, versioned selectors, pagination handlers, and login flows—platform thinking over one-off scrapers
+- **🔔 Monitoring & Alerting**: Alert callbacks on failure thresholds; rollback hooks for failed extraction runs
+- **🔌 MCP Extensibility**: Pluggable backends for Scrapy MCP Server, Playwright MCP, or LLM-based semantic extraction
 - **🔄 Intelligent Retry Logic**: Automatic retries with exponential backoff and jitter
 - **⚡ Rate Limiting**: Token bucket algorithm to respect server limits
 - **🕵️ Anti-Detection**: Stealth mode, user agent rotation, and proxy support
@@ -29,6 +25,10 @@ ScrapeFlow is a production-ready Python library that transforms Playwright into
 - **🔧 Error Handling**: Comprehensive error classification and recovery
 - **📝 Type Hints**: Full type support for better IDE experience
 
+<p align="center">
+  <img src="docs/scrapeflow-architecture.png" alt="ScrapeFlow Architecture" width="800">
+</p>
+
 ## 📦 Installation
 
 ```bash
@@ -381,6 +381,63 @@ config = ScrapeFlowConfig(
 )
 ```
 
+### Specification-Driven Extraction (Pydantic)
+
+**Use Case:** Declarative extraction with validation—fields, types, and rules in specs, not fragile XPaths.
+
+```python
+from pydantic import BaseModel
+from scrapeflow import ScrapeFlow, SpecificationExtractor
+from scrapeflow.specifications import FieldSpec, ItemSpec, ProductPriceSpec
+
+# Model for list of products
+class BookListing(BaseModel):
+    books: list[ProductPriceSpec]
+
+# Schema maps fields to selectors
+schema = {
+    "books": ItemSpec(
+        items_selector="article.product_pod",
+        fields={
+            "title": FieldSpec(selector="h3 a"),
+            "price": FieldSpec(selector=".price_color"),
+            "availability": FieldSpec(selector=".instock.availability", default=""),
+            "url": FieldSpec(selector="h3 a", type="attribute", attribute="href"),
+        },
+    )
+}
+
+async with ScrapeFlow() as scraper:
+    await scraper.navigate("https://books.toscrape.com/")
+    extractor = SpecificationExtractor(BookListing, schema=schema)
+    # Extract and validate in one step
+    data = await extractor.extract(scraper.page)
+    for book in data.books:
+        print(f"{book.title}: {book.price}")
+```
+
+### Ethical Crawling & robots.txt
+
+**Use Case:** GDPR/CCPA compliance and robots.txt respect built into the specification layer.
+
+```python
+from scrapeflow import ScrapeFlow
+from scrapeflow.config import ScrapeFlowConfig, EthicalCrawlingConfig
+
+config = ScrapeFlowConfig(
+    ethical_crawling=EthicalCrawlingConfig(
+        respect_robots_txt=True,      # Check robots.txt before each request
+        user_agent_for_robots="ScrapeFlow",
+        anonymize_ip=True,            # GDPR: minimize personal data
+        data_retention_days=30,       # Document retention policy
+    )
+)
+
+async with ScrapeFlow(config) as scraper:
+    # navigate() automatically checks robots.txt
+    await scraper.navigate("https://example.com/page")
+```
+
 ### Anti-Detection
 
 **Use Case:** Scraping protected e-commerce sites that block automated access.
@@ -806,16 +863,26 @@ ScrapeFlow is built with a modular architecture:
 ```
 scrapeflow/
 ├── engine.py          # Main ScrapeFlow engine
-├── workflow.py        # Workflow definition and execution
-├── config.py          # Configuration classes
+├── ports.py           # Protocols for dependency inversion
+├── browser_runtime.py # Playwright runtime adapter
+├── workflow.py        # Workflow definition entities
+├── workflow_executor.py # Workflow execution service
+├── config.py          # Configuration classes (incl. EthicalCrawlingConfig)
+├── specifications.py  # Pydantic specification-driven extraction
+├── schema_library.py  # Reusable schema definitions
+├── robots.py          # robots.txt parsing and enforcement
+├── registry.py        # Shared selector/component registry
+├── mcp_backend.py     # MCP integration extensibility
 ├── anti_detection.py  # Anti-detection utilities
 ├── rate_limiter.py    # Rate limiting implementation
 ├── retry.py           # Retry logic and error classification
-├── monitoring.py      # Metrics and logging
+├── monitoring.py      # Metrics, logging, alerting
 ├── extractors.py      # Data extraction utilities
 └── exceptions.py      # Custom exceptions
 ```
 
+For deeper design details, see [`ARCHITECTURE.md`](ARCHITECTURE.md).
+
 ## 🤝 Contributing
 
 Contributions are welcome! Please feel free to submit a Pull Request.

scrapeflow_py-0.2.0/examples/authenticated_quotes_example.py

@@ -0,0 +1,77 @@
+"""
+Authenticated extraction example using quotes.toscrape.com.
+
+Demonstrates:
+- login workflow with username/password
+- authenticated data extraction using SpecificationExtractor
+- robots.txt-aware navigation
+"""
+
+import asyncio
+from pydantic import BaseModel
+
+from scrapeflow import (
+    ScrapeFlow,
+    SpecificationExtractor,
+    get_registry,
+    register_quotes_login_handler,
+)
+from scrapeflow.config import ScrapeFlowConfig, EthicalCrawlingConfig
+from scrapeflow.specifications import FieldSpec, ItemSpec
+
+
+class QuoteItem(BaseModel):
+    text: str
+    author: str
+
+
+class QuotesPage(BaseModel):
+    quotes: list[QuoteItem]
+
+
+async def run_authenticated_scrape(username: str, password: str) -> None:
+    schema = {
+        "quotes": ItemSpec(
+            items_selector=".quote",
+            fields={
+                "text": FieldSpec(selector=".text"),
+                "author": FieldSpec(selector=".author"),
+            },
+        )
+    }
+
+    config = ScrapeFlowConfig(
+        ethical_crawling=EthicalCrawlingConfig(
+            respect_robots_txt=True,
+            user_agent_for_robots="ScrapeFlow",
+        )
+    )
+
+    async with ScrapeFlow(config) as scraper:
+        register_quotes_login_handler(get_registry())
+        handler = get_registry().get_login("quotes_login")
+        if handler is None:
+            raise RuntimeError("Missing quotes_login handler in registry")
+
+        is_logged_in = await scraper.login_with_handler(
+            login_url="https://quotes.toscrape.com/login",
+            username=username,
+            password=password,
+            handler=handler,
+            timeout=10000,
+        )
+        if not is_logged_in:
+            error_text = await scraper.page.locator(".error").first.text_content()
+            raise RuntimeError(f"Login failed: {(error_text or '').strip()}")
+
+        extractor = SpecificationExtractor(QuotesPage, schema=schema)
+        data = await extractor.extract(scraper.page)
+
+        print(f"Authenticated login: {is_logged_in}")
+        print(f"Quotes extracted: {len(data.quotes)}")
+        for idx, quote in enumerate(data.quotes[:3], start=1):
+            print(f"{idx}. {quote.author}: {quote.text[:80]}...")
+
+
+if __name__ == "__main__":
+    asyncio.run(run_authenticated_scrape(username="admin", password="amdin"))

scrapeflow_py-0.2.0/examples/specification_driven_example.py

@@ -0,0 +1,80 @@
+"""
+Specification-driven extraction example - aligns with PST.AG job requirements.
+
+Demonstrates:
+- Pydantic specification-driven extraction
+- robots.txt compliance
+- Ethical crawling (GDPR/CCPA config)
+- Shared selector registry
+- Monitoring with alerting
+"""
+
+import asyncio
+from pydantic import BaseModel, Field
+
+from scrapeflow import ScrapeFlow, SpecificationExtractor
+from scrapeflow.config import (
+    ScrapeFlowConfig,
+    EthicalCrawlingConfig,
+    RateLimitConfig,
+)
+from scrapeflow.specifications import FieldSpec, ItemSpec, JobListingSpec
+from scrapeflow.registry import get_registry, register_job_listing_schema
+
+
+class JobListingsPage(BaseModel):
+    """Output model for job listings extraction."""
+
+    jobs: list[JobListingSpec]
+
+
+async def scrape_job_listings():
+    """Scrape job listings using specification-driven extraction."""
+    # Ethical crawling: robots.txt, rate limiting
+    config = ScrapeFlowConfig(
+        ethical_crawling=EthicalCrawlingConfig(
+            respect_robots_txt=True,
+            user_agent_for_robots="ScrapeFlow",
+        ),
+        rate_limit=RateLimitConfig(requests_per_second=1.0),
+    )
+
+    schema = {
+        "jobs": ItemSpec(
+            items_selector=".job-card, .job-listing, article.job, .job-item, .quote",
+            fields={
+                "title": FieldSpec(selector="h2 a, .job-title, .title a, .text"),
+                "company": FieldSpec(
+                    selector=".company, .employer, .author",
+                    default="",
+                ),
+                "location": FieldSpec(
+                    selector=".location, .job-location",
+                    default="",
+                ),
+                "url": FieldSpec(
+                    selector="h2 a, .job-title a, a",
+                    type="attribute",
+                    attribute="href",
+                ),
+            },
+        )
+    }
+
+    async with ScrapeFlow(config) as scraper:
+        # navigate() checks robots.txt automatically
+        await scraper.navigate("https://quotes.toscrape.com/")
+
+        extractor = SpecificationExtractor(JobListingsPage, schema=schema)
+        data = await extractor.extract(scraper.page)
+
+        print(f"Extracted {len(data.jobs)} items (validated via Pydantic)")
+        for i, job in enumerate(data.jobs[:3]):
+            print(f"  {i+1}. {job.title[:50]}... | {job.company or 'N/A'}")
+
+        metrics = scraper.get_metrics()
+        print(f"\nMetrics: {metrics.total_requests} requests, {metrics.get_success_rate():.1f}% success")
+
+
+if __name__ == "__main__":
+    asyncio.run(scrape_job_listings())

{scrapeflow_py-0.1.2 → scrapeflow_py-0.2.0}/requirements.txt

@@ -1,4 +1,5 @@
 playwright>=1.40.0
 aiohttp>=3.9.0
 tenacity>=8.2.0
+pydantic>=2.0.0
 

scrapeflow_py-0.2.0/scrapeflow/__init__.py

@@ -0,0 +1,71 @@
+"""
+ScrapeFlow - An opinionated scraping workflow engine built on Playwright.
+
+ScrapeFlow provides a robust, production-ready framework for web scraping
+with built-in error handling, retry logic, anti-detection, and workflow management.
+
+Specification-driven extraction, robots.txt compliance, and ethical crawling
+(GDPR/CCPA) are built into the design.
+"""
+
+from scrapeflow.engine import ScrapeFlow
+from scrapeflow.workflow import Workflow, Step
+from scrapeflow.workflow_executor import WorkflowExecutor
+from scrapeflow.extractors import Extractor, StructuredExtractor
+from scrapeflow.specifications import (
+    FieldSpec,
+    ItemSpec,
+    SpecificationExtractor,
+    ProductPriceSpec,
+    JobListingSpec,
+    TariffCodeSpec,
+)
+from scrapeflow.robots import RobotsChecker
+from scrapeflow.registry import (
+    SelectorRegistry,
+    LoginHandler,
+    get_registry,
+    register_quotes_login_handler,
+)
+from scrapeflow.browser_runtime import PlaywrightBrowserRuntime
+from scrapeflow.mcp_backend import MCPBackend, PlaceholderMCPBackend, create_mcp_backend
+from scrapeflow.exceptions import (
+    ScrapeFlowError,
+    ScrapeFlowRetryError,
+    ScrapeFlowTimeoutError,
+    ScrapeFlowBlockedError,
+    ScrapeFlowValidationError,
+    ScrapeFlowRobotsDisallowedError,
+)
+
+__version__ = "0.2.0"
+__all__ = [
+    "ScrapeFlow",
+    "Workflow",
+    "Step",
+    "WorkflowExecutor",
+    "Extractor",
+    "StructuredExtractor",
+    "FieldSpec",
+    "ItemSpec",
+    "SpecificationExtractor",
+    "ProductPriceSpec",
+    "JobListingSpec",
+    "TariffCodeSpec",
+    "RobotsChecker",
+    "SelectorRegistry",
+    "LoginHandler",
+    "get_registry",
+    "register_quotes_login_handler",
+    "PlaywrightBrowserRuntime",
+    "MCPBackend",
+    "PlaceholderMCPBackend",
+    "create_mcp_backend",
+    "ScrapeFlowError",
+    "ScrapeFlowRetryError",
+    "ScrapeFlowTimeoutError",
+    "ScrapeFlowBlockedError",
+    "ScrapeFlowValidationError",
+    "ScrapeFlowRobotsDisallowedError",
+]
+

scrapeflow_py-0.2.0/scrapeflow/browser_runtime.py

@@ -0,0 +1,96 @@
+"""Playwright browser runtime adapter."""
+
+from typing import Optional
+from playwright.async_api import (
+    async_playwright,
+    Playwright,
+    Browser,
+    BrowserContext,
+    Page,
+)
+
+from scrapeflow.config import ScrapeFlowConfig, BrowserType
+from scrapeflow.anti_detection import AntiDetectionManager
+
+
+class PlaywrightBrowserRuntime:
+    """Infrastructure adapter for Playwright lifecycle and navigation."""
+
+    def __init__(self, config: ScrapeFlowConfig, anti_detection: AntiDetectionManager):
+        self.config = config
+        self.anti_detection = anti_detection
+        self.playwright: Optional[Playwright] = None
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        self._page: Optional[Page] = None
+        self._is_running = False
+
+    @property
+    def page(self) -> Optional[Page]:
+        return self._page
+
+    async def start(self) -> None:
+        if self._is_running:
+            return
+
+        self.playwright = await async_playwright().start()
+        browser_type_map = {
+            BrowserType.CHROMIUM: self.playwright.chromium,
+            BrowserType.FIREFOX: self.playwright.firefox,
+            BrowserType.WEBKIT: self.playwright.webkit,
+        }
+        browser_launcher = browser_type_map[self.config.browser.browser_type]
+
+        proxy = self.config.browser.proxy or self.anti_detection.get_proxy()
+        browser_args = self.config.browser.args.copy()
+        if self.config.anti_detection.stealth_mode:
+            browser_args.extend(
+                [
+                    "--disable-blink-features=AutomationControlled",
+                    "--disable-dev-shm-usage",
+                    "--no-sandbox",
+                ]
+            )
+
+        self.browser = await browser_launcher.launch(
+            headless=self.config.browser.headless,
+            slow_mo=self.config.browser.slow_mo,
+            args=browser_args,
+            proxy=proxy,
+        )
+
+        context_options = {
+            "viewport": {
+                "width": self.config.anti_detection.viewport_width,
+                "height": self.config.anti_detection.viewport_height,
+            },
+            "user_agent": self.anti_detection.get_user_agent(),
+        }
+        self.context = await self.browser.new_context(**context_options)
+        self._page = await self.context.new_page()
+        await self.anti_detection.apply_stealth(self._page)
+        self._page.set_default_timeout(self.config.browser.timeout)
+        self._is_running = True
+
+    async def close(self) -> None:
+        if not self._is_running:
+            return
+
+        if self._page:
+            await self._page.close()
+            self._page = None
+        if self.context:
+            await self.context.close()
+            self.context = None
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None
+        self._is_running = False
+
+    async def goto(self, url: str, wait_until: str, timeout: int) -> None:
+        if not self._page:
+            raise RuntimeError("Browser runtime is not started.")
+        await self._page.goto(url, wait_until=wait_until, timeout=timeout)

{scrapeflow_py-0.1.2 → scrapeflow_py-0.2.0}/scrapeflow/config.py

@@ -61,6 +61,24 @@ class BrowserConfig:
     proxy: Optional[Dict[str, str]] = None
 
 
+@dataclass
+class EthicalCrawlingConfig:
+    """
+    Ethical crawling configuration—GDPR/CCPA baked into specification design.
+
+    Rate limiting, robots.txt respect, and compliance are built into the
+    specification layer, not retrofitted.
+    """
+
+    respect_robots_txt: bool = True
+    user_agent_for_robots: str = "ScrapeFlow"
+    max_pages_per_domain: Optional[int] = None
+    honor_noindex: bool = True
+    data_retention_days: Optional[int] = None  # GDPR: document retention
+    anonymize_ip: bool = False  # GDPR: minimize personal data
+    consent_required: bool = False  # GDPR: require consent for tracking
+
+
 @dataclass
 class ScrapeFlowConfig:
     """Main configuration for ScrapeFlow."""
@@ -69,6 +87,9 @@ class ScrapeFlowConfig:
     retry: RetryConfig = field(default_factory=RetryConfig)
     rate_limit: RateLimitConfig = field(default_factory=RateLimitConfig)
     anti_detection: AntiDetectionConfig = field(default_factory=AntiDetectionConfig)
+    ethical_crawling: EthicalCrawlingConfig = field(
+        default_factory=EthicalCrawlingConfig
+    )
     log_level: str = "INFO"
     log_file: Optional[str] = None
     debug: bool = False