langchain-zenrows 0.1.0__py3-none-any.whl

langchain_zenrows/__init__.py

@@ -0,0 +1,20 @@
+"""LangChain ZenRows integration package.
+
+This package provides integration between LangChain and ZenRows Universal Scraper API,
+enabling powerful web scraping capabilities with anti-bot bypass, JavaScript
+rendering, and geo-targeting features.
+"""
+
+from langchain_zenrows.zenrows_universal_scraper import (
+    ZenRowsUniversalScraper,
+    ZenRowsUniversalScraperAPIWrapper,
+    ZenRowsUniversalScraperInput,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ZenRowsUniversalScraper",
+    "ZenRowsUniversalScraperAPIWrapper",
+    "ZenRowsUniversalScraperInput",
+]

langchain_zenrows/py.typed

@@ -0,0 +1 @@
+# This file indicates that the package supports type hints

langchain_zenrows/zenrows_universal_scraper.py

@@ -0,0 +1,278 @@
+"""ZenRows Universal Scraper API integration for LangChain."""
+
+import json
+import os
+from typing import Any, Dict, Literal, Optional, Type, Union
+
+import requests
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field, field_validator
+
+
+class ZenRowsUniversalScraperInput(BaseModel):
+    """Input schema for ZenRows Universal Scraper API."""
+
+    url: str = Field(description="The URL of the page you want to scrape")
+    js_render: Optional[bool] = Field(
+        default=False,
+        description="Enable JavaScript rendering with a headless browser. Essential for modern web apps, SPAs, and sites with dynamic content.",
+    )
+    js_instructions: Optional[str] = Field(
+        default=None,
+        description="Execute custom JavaScript on the page to interact with elements, scroll, click buttons, or manipulate content.",
+    )
+    premium_proxy: Optional[bool] = Field(
+        default=False,
+        description="Use residential IPs to bypass anti-bot protection. Essential for accessing protected sites.",
+    )
+    proxy_country: Optional[str] = Field(
+        default=None,
+        description="Set the country of the IP used for the request (requires Premium Proxies). Use for accessing geo-restricted content.",
+    )
+    session_id: Optional[int] = Field(
+        default=None,
+        description="Maintain the same IP for multiple requests for up to 10 minutes. Essential for multi-step processes.",
+    )
+    custom_headers: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="Include custom headers in your request to mimic browser behavior.",
+    )
+    wait_for: Optional[str] = Field(
+        default=None,
+        description="Wait for a specific CSS Selector to appear in the DOM before returning content.",
+    )
+    wait: Optional[int] = Field(
+        default=None, description="Wait a fixed amount of milliseconds after page load."
+    )
+    block_resources: Optional[str] = Field(
+        default=None,
+        description="Block specific resources (images, fonts, etc.) from loading to speed up scraping.",
+    )
+    response_type: Optional[Literal["markdown", "plaintext", "pdf"]] = Field(
+        default=None,
+        description="Convert HTML to other formats. Options: markdown, plaintext, pdf.",
+    )
+    css_extractor: Optional[str] = Field(
+        default=None,
+        description="Extract specific elements using CSS selectors (JSON format).",
+    )
+    autoparse: Optional[bool] = Field(
+        default=False, description="Automatically extract structured data from HTML."
+    )
+    screenshot: Optional[str] = Field(
+        default=None, description="Capture an above-the-fold screenshot of the page."
+    )
+    screenshot_fullpage: Optional[str] = Field(
+        default=None, description="Capture a full-page screenshot."
+    )
+    screenshot_selector: Optional[str] = Field(
+        default=None,
+        description="Capture a screenshot of a specific element using CSS Selector.",
+    )
+    original_status: Optional[bool] = Field(
+        default=False,
+        description="Return the original HTTP status code from the target page.",
+    )
+    allowed_status_codes: Optional[str] = Field(
+        default=None,
+        description="Returns the content even if the target page fails with specified status codes. Useful for debugging or when you need content from error pages.",
+    )
+    json_response: Optional[bool] = Field(
+        default=False,
+        description="Capture network requests in JSON format, including XHR or Fetch data. Ideal for intercepting API calls made by the web page.",
+    )
+    screenshot_format: Optional[Literal["png", "jpeg"]] = Field(
+        default=None,
+        description="Choose between png (default) and jpeg formats for screenshots.",
+    )
+    screenshot_quality: Optional[int] = Field(
+        default=None,
+        description="For JPEG format, set quality from 1 to 100. Lower values reduce file size but decrease quality.",
+    )
+    outputs: Optional[str] = Field(
+        default=None,
+        description="Specify which data types to extract from the scraped HTML. Accepted values: emails, phone_numbers, headings, images, audios, videos, links, menus, hashtags, metadata, tables, favicon.",
+    )
+
+    @field_validator("css_extractor")
+    @classmethod
+    def validate_css_extractor(cls, v):
+        """Validate that css_extractor is valid JSON if provided."""
+        if v is not None:
+            try:
+                json.loads(v)
+            except json.JSONDecodeError:
+                raise ValueError("css_extractor must be valid JSON")
+        return v
+
+    @field_validator("proxy_country")
+    @classmethod
+    def validate_proxy_country(cls, v):
+        """Validate that proxy_country is a two-letter country code."""
+        if v is not None and len(v) != 2:
+            raise ValueError("proxy_country must be a two-letter country code")
+        return v
+
+
+class ZenRowsUniversalScraper(BaseTool):
+    """ZenRows Universal Scraper API tool for LangChain.
+
+    This tool provides access to ZenRows' Universal Scraper API, enabling reliable
+    web scraping with advanced features like JavaScript rendering, premium proxies,
+    and anti-bot bypass.
+
+    To use this tool, you must sign up for a ZenRows account and obtain an API key.
+    Visit https://www.zenrows.com/ to get started.
+    """
+
+    name: str = "zenrows_universal_scraper"
+    description: str = (
+        "A powerful web scraping tool that can extract data from websites with "
+        "anti-bot protection, JavaScript rendering, and geo-targeting capabilities. "
+        "Use this when you need to scrape modern websites, bypass bot detection, "
+        "or access geo-restricted content. Supports multiple output formats including "
+        "HTML, Markdown, and screenshots."
+    )
+    args_schema: Type[BaseModel] = ZenRowsUniversalScraperInput
+
+    zenrows_api_key: Optional[str] = None
+    base_url: str = "https://api.zenrows.com/v1/"
+
+    def __init__(self, zenrows_api_key: Optional[str] = None, **kwargs):
+        """Initialize the ZenRows Universal Scraper tool.
+
+        Args:
+            zenrows_api_key: Your ZenRows API key. If not provided, will look for
+                           ZENROWS_API_KEY environment variable.
+            **kwargs: Additional arguments passed to BaseTool.
+        """
+        super().__init__(**kwargs)
+        self.zenrows_api_key = zenrows_api_key or os.environ.get("ZENROWS_API_KEY")
+
+        if not self.zenrows_api_key:
+            raise ValueError(
+                "ZenRows API key is required. Set ZENROWS_API_KEY environment "
+                "variable or pass zenrows_api_key parameter."
+            )
+
+    def _prepare_request_params(
+        self, tool_input: Union[str, Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        """Prepare request parameters for the ZenRows API."""
+        # Handle string input (just URL)
+        if isinstance(tool_input, str):
+            params = {"url": tool_input}
+        else:
+            params = tool_input.copy()
+
+        # Auto-enable js_render for parameters that require JavaScript rendering
+        js_required_params = [
+            "screenshot",
+            "screenshot_fullpage",
+            "screenshot_selector",
+            "js_instructions",
+            "json_response",
+            "wait",
+            "wait_for",
+        ]
+        js_required = any(params.get(param) for param in js_required_params)
+
+        if js_required:
+            # If any parameter requiring JS is provided, enable js_render
+            params["js_render"] = True
+
+        # Special handling for screenshot variants
+        screenshot_variants = ["screenshot_fullpage", "screenshot_selector"]
+        if any(params.get(param) for param in screenshot_variants):
+            # If screenshot_fullpage or screenshot_selector is provided,
+            # also enable the base screenshot parameter
+            params["screenshot"] = "true"
+
+        # Auto-enable premium_proxy when proxy_country is specified
+        if params.get("proxy_country"):
+            params["premium_proxy"] = True
+
+        # Add required API key
+        params["apikey"] = self.zenrows_api_key
+
+        # Handle custom headers according to ZenRows API documentation
+        request_headers = None
+        if "custom_headers" in params and params["custom_headers"]:
+            # Store the headers dictionary for the request
+            request_headers = params["custom_headers"]
+            # Set custom_headers to "true" to enable custom header support in the API
+            params["custom_headers"] = "true"
+        else:
+            # Remove custom_headers if not provided or empty
+            params.pop("custom_headers", None)
+
+        # Remove None values to avoid sending unnecessary parameters
+        params = {k: v for k, v in params.items() if v is not None}
+
+        return params, request_headers
+
+    def _run(self, **kwargs) -> str:
+        """Execute the ZenRows Universal Scraper API request.
+
+        Returns:
+            The scraped content as a string, format depends on response_type parameter.
+        """
+        try:
+            params, request_headers = self._prepare_request_params(kwargs)
+
+            # Make the API request
+            # Note: ZenRows automatically handles User-Agent and other headers
+            response = requests.get(
+                self.base_url,
+                params=params,
+                headers=request_headers,  # Pass custom headers if provided
+            )
+
+            # Check for successful response
+            response.raise_for_status()
+
+            # Handle different response types
+            screenshot_params = [
+                "screenshot",
+                "screenshot_fullpage",
+                "screenshot_selector",
+            ]
+            if any(param in kwargs and kwargs[param] for param in screenshot_params):
+                # For screenshots, return base64 encoded content with metadata
+                return response.content
+
+            # For text content, return the response text
+            return response.text
+
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 401:
+                raise ValueError("Invalid ZenRows API key")
+            elif e.response.status_code == 429:
+                raise ValueError("Rate limit exceeded. Check your ZenRows plan limits.")
+            elif e.response.status_code == 413:
+                raise ValueError(
+                    "Response size too large. Consider using CSS selectors to reduce content."
+                )
+            else:
+                raise ValueError(
+                    f"HTTP error occurred: {e.response.status_code} - {e.response.text}"
+                )
+
+        except requests.exceptions.Timeout:
+            raise ValueError(
+                "Request timed out. The website might be slow or unresponsive."
+            )
+
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Request failed: {str(e)}")
+
+        except Exception as e:
+            raise ValueError(f"Unexpected error: {str(e)}")
+
+    async def _arun(self, **kwargs) -> str:
+        """Async version of _run method."""
+        return self._run(**kwargs)
+
+
+# For backward compatibility and easier imports
+ZenRowsUniversalScraperAPIWrapper = ZenRowsUniversalScraper

langchain_zenrows-0.1.0.dist-info/METADATA

@@ -0,0 +1,243 @@
+Metadata-Version: 2.4
+Name: langchain-zenrows
+Version: 0.1.0
+Summary: A LangChain integration tool that provides reliable web scraping capabilities at any scale using ZenRows' Universal Scraper API
+Author-email: Yuvraj Chandra <support@zenrows.com>
+License: MIT
+Project-URL: Homepage, https://github.com/ZenRows-Hub/langchain-zenrows
+Project-URL: Repository, https://github.com/ZenRows-Hub/langchain-zenrows
+Project-URL: Documentation, https://github.com/ZenRows-Hub/langchain-zenrows
+Keywords: langchain,zenrows,web scraping,llm,ai,ml,data extraction
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain-core>=0.3.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: pydantic>=2.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: pytest-mock>=3.10.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0; extra == "test"
+Requires-Dist: langchain-tests>=0.3.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ruff>=0.1.5; extra == "dev"
+Dynamic: license-file
+
+# langchain-zenrows
+
+The langchain-zenrows integration tool enables LangChain agents to scrape and access web content at any scale using ZenRows' enterprise-grade infrastructure. 
+
+Whether you need to scrape JavaScript-heavy single-page applications, bypass anti-bot systems, access geo-restricted content, or extract structured data at scale, this integration provides the tools and reliability needed for modern AI applications.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [API Reference](#api-reference)
+- [Features](#features)
+- [License](#license)
+
+## Installation
+
+```console
+pip install langchain-zenrows
+```
+
+## Usage
+
+To use the ZenRows Universal Scraper with LangChain, you'll need a ZenRows API key. You can sign up for free at [ZenRows](https://app.zenrows.com/register?prod=universal_scraper).
+
+> For more comprehensive examples and use cases, see the `examples/` folder.
+
+### Basic Usage
+
+```python
+import os
+from langchain_zenrows import ZenRowsUniversalScraper
+
+# Set your ZenRows API key
+os.environ["ZENROWS_API_KEY"] = "<YOUR_ZENROWS_API_KEY>"
+
+# Initialize the tool
+scraper = ZenRowsUniversalScraper()
+
+# Scrape a simple webpage
+result = scraper.invoke({"url": "https://httpbin.io/html"})
+print(result)
+```
+
+### Advanced Usage with Parameters
+
+```python
+import os
+from langchain_zenrows import ZenRowsUniversalScraper
+
+# Set your ZenRows API key
+os.environ["ZENROWS_API_KEY"] = "<YOUR_ZENROWS_API_KEY>"
+
+scraper = ZenRowsUniversalScraper()
+
+# Scrape with JavaScript rendering and premium proxies
+result = scraper.invoke({
+    "url": "https://www.scrapingcourse.com/ecommerce/",
+    "js_render": True,
+    "premium_proxy": True,
+    "proxy_country": "us",
+    "response_type": "markdown",
+    "wait": 2000  # Wait 2 seconds after page load
+})
+
+print(result)
+```
+
+See the [API Reference](#api-reference) section below for more available parameters and customizing scraping requests.
+
+### Using with LangChain Agents
+
+```python
+from langchain_zenrows import ZenRowsUniversalScraper
+from langchain_openai import ChatOpenAI  # or your preferred LLM
+from langgraph.prebuilt import create_react_agent
+import os
+
+# Set your ZenRows API key
+os.environ["ZENROWS_API_KEY"] = "<YOUR_ZENROWS_API_KEY>"
+os.environ["OPENAI_API_KEY"] = "<YOUR_OPEN_AI_API_KEY>"
+
+
+# Initialize components
+llm = ChatOpenAI(model="gpt-4o-mini")
+zenrows_tool = ZenRowsUniversalScraper()
+
+# Create agent
+agent = create_react_agent(llm, [zenrows_tool])
+
+# Use the agent
+result = agent.invoke(
+    {
+        "messages": "Scrape https://news.ycombinator.com/ and list the top 3 stories with title, points, comments, username, and time."
+    }
+)
+
+print("Agent Response:")
+for message in result["messages"]:
+    print(f"{message.content}")
+```
+
+### CSS Extraction
+
+Extract specific data using CSS selectors:
+
+```python
+import json
+import os
+from langchain_zenrows import ZenRowsUniversalScraper
+
+# Set your ZenRows API key
+os.environ["ZENROWS_API_KEY"] = "<YOUR_ZENROWS_API_KEY>"
+
+scraper = ZenRowsUniversalScraper()
+
+# Extract specific elements
+css_selector = json.dumps({
+    "title": "h1",
+    "paragraphs": "p"
+})
+
+result = scraper.invoke({
+    "url": "https://httpbin.io/html",
+    "css_extractor": css_selector
+})
+```
+
+### Premium Proxy with Geo-targeting
+
+Access geo-restricted content:
+
+```python
+import os
+from langchain_zenrows import ZenRowsUniversalScraper
+
+# Set your ZenRows API key
+os.environ["ZENROWS_API_KEY"] = "<YOUR_ZENROWS_API_KEY>"
+
+scraper = ZenRowsUniversalScraper()
+
+# Check your IP location
+result = scraper.invoke({
+    "url": "https://httpbin.io/ip",
+    "premium_proxy": True,
+    "proxy_country": "us"
+})
+print(result)  # Shows the US IP being used
+```
+
+## API Reference
+
+### ZenRowsUniversalScraper
+
+Main tool class for web scraping with ZenRows.
+
+**Parameters:**
+
+- `zenrows_api_key` (str, optional): Your ZenRows API key. If not provided, looks for `ZENROWS_API_KEY` environment variable.
+
+**Input Schema:**
+
+For complete parameter documentation and details, see the [official ZenRows API Reference](https://docs.zenrows.com/universal-scraper-api/api-reference#parameter-overview).
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `url` | str | **Required.** The URL to scrape |
+| `js_render` | bool | Enable JavaScript rendering with a headless browser. Essential for modern web apps, SPAs, and sites with dynamic content (default: False) |
+| `js_instructions` | str | Execute custom JavaScript on the page to interact with elements, scroll, click buttons, or manipulate content |
+| `premium_proxy` | bool | Use residential IPs to bypass anti-bot protection. Essential for accessing protected sites (default: False) |
+| `proxy_country` | str | Set the country of the IP used for the request. Use for accessing geo-restricted content. Two-letter country code |
+| `session_id` | int | Maintain the same IP for multiple requests for up to 10 minutes. Essential for multi-step processes |
+| `custom_headers` | dict | Include custom headers in your request to mimic browser behavior |
+| `wait_for` | str | Wait for a specific CSS Selector to appear in the DOM before returning content |
+| `wait` | int | Wait a fixed amount of milliseconds after page load |
+| `block_resources` | str | Block specific resources (images, fonts, etc.) from loading to speed up scraping |
+| `response_type` | str | Convert HTML to other formats. Options: "markdown", "plaintext", "pdf" |
+| `css_extractor` | str | Extract specific elements using CSS selectors (JSON format) |
+| `autoparse` | bool | Automatically extract structured data from HTML (default: False) |
+| `screenshot` | str | Capture an above-the-fold screenshot of the page (default: "false") |
+| `screenshot_fullpage` | str | Capture a full-page screenshot (default: "false") |
+| `screenshot_selector` | str | Capture a screenshot of a specific element using CSS Selector |
+| `screenshot_format` | str | Choose between "png" (default) and "jpeg" formats for screenshots |
+| `screenshot_quality` | int | For JPEG format, set quality from 1 to 100. Lower values reduce file size but decrease quality |
+| `original_status` | bool | Return the original HTTP status code from the target page (default: False) |
+| `allowed_status_codes` | str | Returns the content even if the target page fails with specified status codes. Useful for debugging or when you need content from error pages |
+| `json_response` | bool | Capture network requests in JSON format, including XHR or Fetch data. Ideal for intercepting API calls made by the web page (default: False) |
+| `outputs` | str | Specify which data types to extract from the scraped HTML. Accepted values: emails, phone_numbers, headings, images, audios, videos, links, menus, hashtags, metadata, tables, favicon |
+
+## Features
+
+- **JavaScript Rendering**: Scrape modern SPAs and dynamic content
+- **Anti-Bot Bypass**: Bypass sophisticated bot detection systems
+- **Geo-Targeting**: Access region-specific content with 190+ countries
+- **Multiple Output Formats**: HTML, Markdown, Plaintext, PDF, Screenshots
+- **CSS Extraction**: Target specific data with CSS selectors
+- **Structured Data Extraction**: Automatically extract emails, phone numbers, links, and other data types
+- **Session Management**: Maintain consistent sessions across requests
+- **Wait Conditions**: Smart waiting for dynamic content
+- **Premium Proxies**: 55M+ residential IPs for maximum success rates
+
+## License
+
+`langchain-zenrows` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license.
+
+## Support
+
+- [ZenRows Documentation](https://docs.zenrows.com/)
+- [LangChain Documentation](https://python.langchain.com/)

langchain_zenrows-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+langchain_zenrows/__init__.py,sha256=DzYCTVQbwmqU_pvUMyUTSPfCksSQ5sFHv6yaygR07fI,559
+langchain_zenrows/py.typed,sha256=q6n3HRnk8qP3ZQEZ6rlHfzcbJHLhoCbNBdY6edgwavo,58
+langchain_zenrows/zenrows_universal_scraper.py,sha256=Cfm1rJQPS1fmf8U1pEN8O3qlKeQS7A7V9kY1W0YmA-s,11347
+langchain_zenrows-0.1.0.dist-info/licenses/LICENSE,sha256=Ed8oGcW7VbSLrdRVbpgMzLQEaqlgGmBHH36KiW6kS3Y,1063
+langchain_zenrows-0.1.0.dist-info/METADATA,sha256=SJCxfJXG3btwYPalTlOVE7zSvlGerH3wheOBgnurSZ0,9472
+langchain_zenrows-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+langchain_zenrows-0.1.0.dist-info/top_level.txt,sha256=L2wYN-NsLb7r9wYqAEpCWYMz4lB3S6xi4O2WILZ42ZM,18
+langchain_zenrows-0.1.0.dist-info/RECORD,,

langchain_zenrows-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

langchain_zenrows-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ZenRows
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_zenrows-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+langchain_zenrows