oxylabs-mcp 0.1.0__tar.gz

oxylabs_mcp-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.envrc

oxylabs_mcp-0.1.0/Dockerfile

@@ -0,0 +1,24 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
+ENV UV_CACHE_DIR=/opt/uv-cache/
+
+RUN apt-get update && apt-get install -y --no-install-recommends git
+
+WORKDIR /app
+
+RUN --mount=type=cache,target=UV_CACHE_DIR \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev --no-editable
+
+ADD . /app
+
+RUN --mount=type=cache,target=UV_CACHE_DIR \
+    uv sync --frozen --no-dev --no-editable
+
+# Add virtual environment to PATH
+ENV PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["oxylabs-mcp"]

oxylabs_mcp-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Oxylabs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

oxylabs_mcp-0.1.0/Makefile

@@ -0,0 +1,34 @@
+src_dir := src/
+test_dir := tests/
+current_dir = $(shell pwd)
+
+python ?= python3
+virtualenv_dir ?= .venv
+
+.PHONY: install_deps
+install_deps: $(virtualenv_dir)
+	uv sync
+
+.PHONY: lint
+lint: install_deps
+	uv run mypy $(src_dir)
+	uv run ruff check
+
+.PHONY: format
+format: $(virtualenv_dir)
+	uv run ruff check --fix .
+
+.PHONY: test
+test: install_deps
+	uv run pytest ./tests
+
+.PHONY: run
+run: install_deps
+	npx @modelcontextprotocol/inspector \
+      uv \
+      --directory $(current_dir) \
+      run \
+      oxylabs-mcp
+
+$(virtualenv_dir):
+	$(python) -m venv $@ --symlinks

oxylabs_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: oxylabs-mcp
+Version: 0.1.0
+Summary: Oxylabs MCP server
+Project-URL: Homepage, https://github.com/oxylabs/oxylabs-mcp
+Project-URL: Repository, https://github.com/oxylabs/oxylabs-mcp
+Author-email: Augis Braziunas <augis.braziunas@oxylabs.io>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: lxml-html-clean>=0.4.1
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: markdownify>=0.14.1
+Requires-Dist: mcp[cli]>=1.2.0
+Requires-Dist: pydantic>=2.10.5
+Description-Content-Type: text/markdown
+
+# MCP Server for Oxylabs Scraper
+[![smithery badge](https://smithery.ai/badge/@oxylabs/oxylabs-mcp)](https://smithery.ai/server/@oxylabs/oxylabs-mcp)
+
+The scraper tool is an asynchronous utility that leverages the Oxylabs Web Scraper API to fetch and process content from a given URL. This tool is designed to handle various scraping scenarios efficiently, with flexible options for parsing and rendering web pages.
+
+## Key Features
+
+1. **URL Scraping**
+    - Fetches the content of a specified URL using Oxylabs' API.
+
+2. **HTML Parsing**
+    - If parsing is enabled, the content is returned in a processed format, suitable for handling by downstream systems like language models (LLMs).
+    - If parsing is disabled, the tool strips unnecessary HTML tags and converts the cleaned content into a Markdown file for easier readability and compatibility.
+
+3. **JavaScript Rendering**
+    - Offers an option to render pages using a headless browser, which ensures dynamic content (e.g., JavaScript-rendered pages) is included in the output.
+    - Supports the html rendering option or skipping rendering entirely.
+
+## Examples on how to query Claude or other LLM
+- Could you scrape https://oxylabs.io page?
+- Scrape https://www.amazon.de/-/en/Smartphone-Contract-Function-Manufacturer-Exclusive/dp/B0CNKD651V with parse enabled.
+- Scrape https://www.amazon.de/-/en/gp/bestsellers/beauty/ref=zg_bs_nav_beauty_0 with parse and render enabled.
+- Use web unblocker with render to scrape https://oxylabs.io/
+
+## Prerequisites
+### Install uv first.
+https://docs.astral.sh/uv/getting-started/installation/#installation-methods
+
+## Setup Intructions
+
+### Installing via Smithery
+
+To install Oxylabs MCP server for Claude Desktop automatically via [Smithery](https://smithery.ai/server/@oxylabs/oxylabs-mcp):
+
+```bash
+npx -y @smithery/cli install @oxylabs/oxylabs-mcp --client claude
+```
+
+
+### Setup with Claude Desktop
+```json
+# claude_desktop_config.json
+# Can find location through:
+# Claude -> Settings -> Developer -> Edit Config
+{
+  "mcpServers": {
+    "oxylabs_scraper": {
+      "command": "uvx",
+      "args": ["oxylabs-mcp"],
+      "env": {
+        "OXYLABS_USERNAME": "YOUR_USERNAME_HERE",
+        "OXYLABS_PASSWORD": "YOUR_PASSWORD_HERE",
+      }
+    }
+  }
+}
+```
+
+## Local/Dev Setup Instructions
+### Clone repo
+`git clone <git:url>`
+### Install dependencies
+Install MCP server dependencies:
+```bash
+cd mcp-server-oxylabs
+
+# Create virtual environment and activate it
+uv venv
+
+source .venv/bin/activate # MacOS/Linux
+# OR
+.venv/Scripts/activate # Windows
+
+# Install dependencies
+uv sync
+```
+### Setup with Claude Desktop
+```json
+# claude_desktop_config.json
+# Can find location through:
+# Claude -> Settings -> Developer -> Edit Config
+{
+  "mcpServers": {
+    "oxylabs_scraper": {
+      "command": "uv",
+      "args": [
+        "--directory",
+        "/<Absolute-path-to-folder>/oxylabs-mcp",
+        "run",
+        "oxylabs-mcp"
+      ],
+      "env": {
+        "OXYLABS_USERNAME": "YOUR_USERNAME_HERE",
+        "OXYLABS_PASSWORD": "YOUR_PASSWORD_HERE"
+      }
+    }
+  }
+}
+```
+
+### Debugging
+Run:
+```bash
+make run
+```
+Then access MCP Inspector at `http://localhost:5173`. You may need to add your username and password in the environment variables in the inspector under `OXYLABS_USERNAME` and `OXYLABS_PASSWORD`.
+
+---
+**NOTE**
+
+Usage with the MCP inspector is affected by an ongoing issue with the Python SDK for MCP, see: https://github.com/modelcontextprotocol/python-sdk/pull/85. For Claude, a forked version of the SDK is used as a temporary fix.
+
+---
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## About Oxylabs
+
+Established in 2015, Oxylabs are a market-leading web intelligence collection
+platform, driven by the highest business, ethics, and compliance standards,
+enabling companies worldwide to unlock data-driven insights.
+
+[![image](https://oxylabs.io/images/og-image.png)](https://oxylabs.io/)

oxylabs_mcp-0.1.0/README.md

@@ -0,0 +1,125 @@
+# MCP Server for Oxylabs Scraper
+[![smithery badge](https://smithery.ai/badge/@oxylabs/oxylabs-mcp)](https://smithery.ai/server/@oxylabs/oxylabs-mcp)
+
+The scraper tool is an asynchronous utility that leverages the Oxylabs Web Scraper API to fetch and process content from a given URL. This tool is designed to handle various scraping scenarios efficiently, with flexible options for parsing and rendering web pages.
+
+## Key Features
+
+1. **URL Scraping**
+    - Fetches the content of a specified URL using Oxylabs' API.
+
+2. **HTML Parsing**
+    - If parsing is enabled, the content is returned in a processed format, suitable for handling by downstream systems like language models (LLMs).
+    - If parsing is disabled, the tool strips unnecessary HTML tags and converts the cleaned content into a Markdown file for easier readability and compatibility.
+
+3. **JavaScript Rendering**
+    - Offers an option to render pages using a headless browser, which ensures dynamic content (e.g., JavaScript-rendered pages) is included in the output.
+    - Supports the html rendering option or skipping rendering entirely.
+
+## Examples on how to query Claude or other LLM
+- Could you scrape https://oxylabs.io page?
+- Scrape https://www.amazon.de/-/en/Smartphone-Contract-Function-Manufacturer-Exclusive/dp/B0CNKD651V with parse enabled.
+- Scrape https://www.amazon.de/-/en/gp/bestsellers/beauty/ref=zg_bs_nav_beauty_0 with parse and render enabled.
+- Use web unblocker with render to scrape https://oxylabs.io/
+
+## Prerequisites
+### Install uv first.
+https://docs.astral.sh/uv/getting-started/installation/#installation-methods
+
+## Setup Intructions
+
+### Installing via Smithery
+
+To install Oxylabs MCP server for Claude Desktop automatically via [Smithery](https://smithery.ai/server/@oxylabs/oxylabs-mcp):
+
+```bash
+npx -y @smithery/cli install @oxylabs/oxylabs-mcp --client claude
+```
+
+
+### Setup with Claude Desktop
+```json
+# claude_desktop_config.json
+# Can find location through:
+# Claude -> Settings -> Developer -> Edit Config
+{
+  "mcpServers": {
+    "oxylabs_scraper": {
+      "command": "uvx",
+      "args": ["oxylabs-mcp"],
+      "env": {
+        "OXYLABS_USERNAME": "YOUR_USERNAME_HERE",
+        "OXYLABS_PASSWORD": "YOUR_PASSWORD_HERE",
+      }
+    }
+  }
+}
+```
+
+## Local/Dev Setup Instructions
+### Clone repo
+`git clone <git:url>`
+### Install dependencies
+Install MCP server dependencies:
+```bash
+cd mcp-server-oxylabs
+
+# Create virtual environment and activate it
+uv venv
+
+source .venv/bin/activate # MacOS/Linux
+# OR
+.venv/Scripts/activate # Windows
+
+# Install dependencies
+uv sync
+```
+### Setup with Claude Desktop
+```json
+# claude_desktop_config.json
+# Can find location through:
+# Claude -> Settings -> Developer -> Edit Config
+{
+  "mcpServers": {
+    "oxylabs_scraper": {
+      "command": "uv",
+      "args": [
+        "--directory",
+        "/<Absolute-path-to-folder>/oxylabs-mcp",
+        "run",
+        "oxylabs-mcp"
+      ],
+      "env": {
+        "OXYLABS_USERNAME": "YOUR_USERNAME_HERE",
+        "OXYLABS_PASSWORD": "YOUR_PASSWORD_HERE"
+      }
+    }
+  }
+}
+```
+
+### Debugging
+Run:
+```bash
+make run
+```
+Then access MCP Inspector at `http://localhost:5173`. You may need to add your username and password in the environment variables in the inspector under `OXYLABS_USERNAME` and `OXYLABS_PASSWORD`.
+
+---
+**NOTE**
+
+Usage with the MCP inspector is affected by an ongoing issue with the Python SDK for MCP, see: https://github.com/modelcontextprotocol/python-sdk/pull/85. For Claude, a forked version of the SDK is used as a temporary fix.
+
+---
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## About Oxylabs
+
+Established in 2015, Oxylabs are a market-leading web intelligence collection
+platform, driven by the highest business, ethics, and compliance standards,
+enabling companies worldwide to unlock data-driven insights.
+
+[![image](https://oxylabs.io/images/og-image.png)](https://oxylabs.io/)

oxylabs_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[project]
+name = "oxylabs-mcp"
+version = "0.1.0"
+description = "Oxylabs MCP server"
+authors = [
+    {name="Augis Braziunas", email="augis.braziunas@oxylabs.io"},
+]
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Development Status :: 4 - Beta",
+    "Operating System :: OS Independent",
+]
+
+license = "MIT"
+license-files = ["LICEN[CS]E*"]
+
+dependencies = [
+    "httpx>=0.28.1",
+    "lxml>=5.3.0",
+    "lxml-html-clean>=0.4.1",
+    "markdownify>=0.14.1",
+    "mcp[cli]>=1.2.0",
+    "pydantic>=2.10.5",
+]
+
+[dependency-groups]
+dev = [
+    "mypy>=1.14.1",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.2",
+    "pytest-mock>=3.14.0",
+    "ruff>=0.9.1",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+# Fork with minimal fix for Claude disconnection errors
+# When https://github.com/modelcontextprotocol/python-sdk/pull/85 merged, change back to official lib.
+[tool.uv.sources]
+mcp = { git = "https://github.com/txbm/python-sdk", rev = "19185626d57fdd670219784d10859c7d0a302c2e" }
+
+[project.scripts]
+oxylabs-mcp = "oxylabs_mcp:main"
+
+[project.urls]
+Homepage = "https://github.com/oxylabs/oxylabs-mcp"
+Repository = "https://github.com/oxylabs/oxylabs-mcp"

oxylabs_mcp-0.1.0/smithery.yaml

@@ -0,0 +1,21 @@
+# Smithery configuration file: https://smithery.ai/docs/deployments
+
+startCommand:
+  type: stdio
+  configSchema:
+    # JSON Schema defining the configuration options for the MCP.
+    type: object
+    required:
+      - oxylabsUsername
+      - oxylabsPassword
+    properties:
+      oxylabsUsername:
+        type: string
+        description: The username for Oxylabs API authentication.
+      oxylabsPassword:
+        type: string
+        description: The password for Oxylabs API authentication.
+  commandFunction:
+    # A function that produces the CLI command to start the MCP on stdio.
+    |-
+    config => ({ command: 'oxylabs-mcp', env: { OXYLABS_USERNAME: config.oxylabsUsername, OXYLABS_PASSWORD: config.oxylabsPassword } })

oxylabs_mcp-0.1.0/src/oxylabs_mcp/__init__.py

@@ -0,0 +1,10 @@
+from . import server
+
+
+def main():
+    """Main entry point for the package."""
+    server.main()
+
+
+# Optionally expose other important items at package level
+__all__ = ["main", "server"]

oxylabs_mcp-0.1.0/src/oxylabs_mcp/server.py

@@ -0,0 +1,132 @@
+from typing import Annotated, Any, Literal
+from httpx import AsyncClient, BasicAuth, Timeout, HTTPStatusError, RequestError
+from mcp.server.fastmcp import FastMCP
+from dotenv import load_dotenv
+from pydantic import Field
+
+from oxylabs_mcp.utils import convert_html_to_md, get_auth_from_env, strip_html
+
+OXYLABS_SCRAPER_URL = "https://realtime.oxylabs.io/v1/queries"
+REQUEST_TIMEOUT = 100
+
+mcp = FastMCP("oxylabs_mcp", dependencies=["mcp", "httpx"])
+load_dotenv()
+
+
+@mcp.tool(name="oxylabs_scraper", description="Scrape url using Oxylabs Web Api")
+async def scrape_url(
+    url: Annotated[str, Field(description="Url to scrape")],
+    parse: Annotated[
+        bool | None,
+        Field(
+            description="Should result be parsed. "
+            "If result should not be parsed then html "
+            "will be stripped and converted to markdown file"
+        )
+    ] = None,
+    render: Annotated[
+        Literal["html", "None"] | None,
+        Field(
+            description="Whether a headless browser should be used "
+            "to render the page. See: "
+            "https://developers.oxylabs.io/scraper-apis/web-scraper-api/features/javascript-rendering "
+            "`html` will return rendered html page "
+            "`None` will not use render for scraping."
+        )
+    ] = None
+) -> str:
+    """Scrape Url using Oxylabs scraper api"""
+    username, password = get_auth_from_env()
+
+    async with AsyncClient(
+        auth=BasicAuth(username=username, password=password),
+        timeout=Timeout(REQUEST_TIMEOUT)
+    ) as client:
+        try:
+            json: dict[str, Any] = {"url": url}
+            if parse:
+                json["parse"] = bool(parse)
+            if render and render != "None":
+                json["render"] = render
+
+            response = await client.post(
+                OXYLABS_SCRAPER_URL,
+                json=json,
+            )
+            response.raise_for_status()
+
+            if not bool(parse):
+                striped_html = strip_html(
+                    str(response.json()["results"][0]["content"])
+                )
+                return convert_html_to_md(striped_html)
+            return str(response.json()["results"][0]["content"])
+        except HTTPStatusError as e:
+            return (
+                "HTTP error during POST request: "
+                f"{e.response.status_code} - {e.response.text}"
+            )
+        except RequestError as e:
+            return f"Request error during POST request: {e}"
+        except Exception as e:
+            return f"Error: {str(e) or repr(e)}"
+
+
+@mcp.tool(
+    name="oxylabs_web_unblocker",
+    description="Scrape url using Oxylabs Web Unblocker",
+)
+async def scrape_with_web_unblocker(
+    url: Annotated[str, Field(description="Url to scrape with web unblocker")],
+    render: Annotated[
+        Literal["html", "None"] | None,
+        Field(
+            description="Whether a headless browser should be used "
+            "to render the page. See: "
+            "https://developers.oxylabs.io/advanced-proxy-solutions/web-unblocker/headless-browser/javascript-rendering "
+            "`html` will return rendered html page "
+            "`None` will not use render for scraping."
+        )
+    ] = None
+) -> str:
+    """Web Unblocker is an AI-powered proxy solution.
+
+    This tool manages the unblocking process to extract public data
+    even from the most difficult websites.
+    """
+    username, password = get_auth_from_env()
+
+    proxy = f"http://{username}:{password}@unblock.oxylabs.io:60000"
+
+    headers: dict[str, Any] = {}
+    if render and render != "None":
+        headers["X-Oxylabs-Render"] = render
+
+    async with AsyncClient(
+        timeout=Timeout(REQUEST_TIMEOUT),
+        verify=False,
+        proxy=proxy,
+        headers=headers
+    ) as client:
+        try:
+            response = await client.get(url)
+            response.raise_for_status()
+            striped_html = strip_html(response.text)
+            return convert_html_to_md(striped_html)
+        except HTTPStatusError as e:
+            return (
+                "HTTP error during POST request: "
+                f"{e.response.status_code} - {e.response.text}"
+            )
+        except RequestError as e:
+            return f"Request error during POST request: {e}"
+        except Exception as e:
+            return f"Error: {str(e) or repr(e)}"
+
+
+def main():
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

oxylabs_mcp-0.1.0/src/oxylabs_mcp/utils.py

@@ -0,0 +1,96 @@
+import os
+from lxml.html.clean import Cleaner
+from lxml.html import defs, fromstring, tostring
+import re
+from markdownify import markdownify as md
+
+
+def get_auth_from_env() -> tuple[str, str]:
+    """Gets Username and password from enviroment variables"""
+    USERNAME = os.getenv('OXYLABS_USERNAME')
+    PASSWORD = os.getenv('OXYLABS_PASSWORD')
+
+    if not USERNAME or not PASSWORD:
+        raise ValueError(
+            "OXYLABS_USERNAME and OXYLABS_PASSWORD "
+            "must be set in the environment variables."
+        )
+    return USERNAME, PASSWORD
+
+
+def clean_html(html: str):
+    cleaner = Cleaner(
+        scripts=True,
+        javascript=True,
+        style=True,
+        remove_tags=[],
+        kill_tags=["nav", "svg", "footer", "noscript", "script", "form"],
+        safe_attrs=list(defs.safe_attrs) + ["idx"],
+        comments=True,
+        inline_style=True,
+        links=True,
+        meta=False,
+        page_structure=False,
+        embedded=True,
+        frames=False,
+        forms=False,
+        annoying_tags=False,
+    )
+    return cleaner.clean_html(html)
+
+
+def strip_html(html: str) -> str:
+    """
+    Cleans and simplifies an HTML string by removing unwanted elements,
+    attributes, and redundant content.
+
+    Args:
+        html (str): The input HTML string.
+
+    Returns:
+        str: The cleaned and simplified HTML string.
+    """
+    cleaned_html = clean_html(html)
+    html_tree = fromstring(cleaned_html)
+
+    for element in html_tree.iter():
+        # Remove style attributes.
+        if "style" in element.attrib:
+            del element.attrib["style"]
+
+        # Remove elements that have no attributes, no content and no children.
+        if (
+            (
+                not element.attrib
+                or (len(element.attrib) == 1 and "idx" in element.attrib)
+            )
+            and not element.getchildren()
+            and (not element.text or not element.text.strip())
+            and (not element.tail or not element.tail.strip())
+        ):
+            parent = element.getparent()
+            if parent is not None:
+                parent.remove(element)
+
+    # Remove elements with footer and hidden in class or id
+    xpath_query = (
+        ".//*[contains(@class, 'footer') or contains(@id, 'footer') or "
+        "contains(@class, 'hidden') or contains(@id, 'hidden')]")
+    elements_to_remove = html_tree.xpath(xpath_query)
+    for element in elements_to_remove:
+        parent = element.getparent()
+        if parent is not None:
+            parent.remove(element)
+
+    # Serialize the HTML tree back to a string
+    stripped_html = tostring(html_tree, encoding="unicode")
+    # Previous cleaning produces empty spaces.
+    # Replace multiple spaces with an single one
+    stripped_html = re.sub(r"\s{2,}", " ", stripped_html)
+    # Replace consecutive newlines with an empty string
+    stripped_html = re.sub(r"\n{2,}", "", stripped_html)
+    return stripped_html
+
+
+def convert_html_to_md(html: str) -> str:
+    return md(html)

oxylabs_mcp-0.1.0/tests/fixtures/after_strip.html

@@ -0,0 +1,2 @@
+<html> <body> <div class="content"> <p>Welcome to my website</p> </div> <div class="other"> <p>Visible content</p> </div> </body>
+</html>