screaming-frog-mcp 0.1.0__tar.gz

screaming_frog_mcp-0.1.0/.env.example

@@ -0,0 +1,9 @@
+# Path to the Screaming Frog CLI executable.
+# macOS default shown below. Adjust for your OS.
+SF_CLI_PATH=/Applications/Screaming Frog SEO Spider.app/Contents/MacOS/ScreamingFrogSEOSpiderLauncher
+
+# Linux example:
+# SF_CLI_PATH=/usr/bin/screamingfrogseospider
+
+# Windows example:
+# SF_CLI_PATH=C:\Program Files (x86)\Screaming Frog SEO Spider\ScreamingFrogSEOSpiderCli.exe

screaming_frog_mcp-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+.DS_Store
+marketing/
+dist/
+build/
+*.egg-info/

screaming_frog_mcp-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Boaz Sasson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

screaming_frog_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: screaming-frog-mcp
+Version: 0.1.0
+Summary: MCP server for Screaming Frog SEO Spider — crawl sites, export data, and manage crawl storage via AI assistants
+Project-URL: Homepage, https://github.com/bzsasson/screaming-frog-mcp
+Project-URL: Repository, https://github.com/bzsasson/screaming-frog-mcp
+Project-URL: Issues, https://github.com/bzsasson/screaming-frog-mcp/issues
+Author: Boaz Sasson
+License-Expression: MIT
+License-File: LICENSE
+Keywords: crawl,mcp,model-context-protocol,screaming-frog,seo
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP :: Site Management
+Requires-Python: >=3.10
+Requires-Dist: mcp>=1.26.0
+Requires-Dist: python-dotenv>=1.0.0
+Description-Content-Type: text/markdown
+
+# Screaming Frog SEO Spider MCP Server
+
+An MCP (Model Context Protocol) server that gives Claude (or any MCP-compatible client) programmatic access to [Screaming Frog SEO Spider](https://www.screamingfrog.co.uk/seo-spider/) — crawl websites, export crawl data, and manage your crawl storage, all from your AI assistant.
+
+## Prerequisites
+
+1. **Screaming Frog SEO Spider** installed on your machine (tested with v23.x, should work with v16+).
+   Download from: https://www.screamingfrog.co.uk/seo-spider/
+
+2. **A valid Screaming Frog license.** The free version has a 500-URL crawl limit. Most MCP features (headless CLI, saving/loading crawls, exports) require a paid license.
+
+3. **Python 3.10+**
+
+## Important: How the Workflow Works
+
+Screaming Frog uses an internal database that can only be accessed by one process at a time. This means:
+
+> **You must close the Screaming Frog GUI before the MCP server can access crawl data.**
+
+The typical workflow is:
+
+1. **Run your crawl** — either through the SF GUI (with all your custom settings, filters, etc.) or via the MCP `crawl_site` tool.
+2. **Close the Screaming Frog GUI** — the GUI locks the crawl database. The MCP server's headless CLI cannot read or export data while the GUI is running.
+3. **Use the MCP tools** — once the GUI is closed, you can list crawls, export data, read CSVs, and more through your AI assistant.
+
+If you forget to close the GUI, the server will detect it and show a clear error message telling you to quit SF first.
+
+## Setup
+
+### Option A: Install from PyPI (recommended)
+
+```bash
+pip install screaming-frog-mcp
+```
+
+Or run directly with `uvx` (no install needed):
+
+```bash
+uvx screaming-frog-mcp
+```
+
+### Option B: Clone and install from source
+
+```bash
+git clone https://github.com/bzsasson/screaming-frog-mcp.git
+cd screaming-frog-mcp
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Configure the CLI path
+
+The default Screaming Frog CLI path works for macOS. If you're on Linux or Windows, set the `SF_CLI_PATH` environment variable:
+
+| OS      | Default Path |
+|---------|-------------|
+| macOS   | `/Applications/Screaming Frog SEO Spider.app/Contents/MacOS/ScreamingFrogSEOSpiderLauncher` |
+| Linux   | `/usr/bin/screamingfrogseospider` |
+| Windows | `C:\Program Files (x86)\Screaming Frog SEO Spider\ScreamingFrogSEOSpiderCli.exe` |
+
+If you cloned the repo, copy `.env.example` to `.env` and edit it.
+
+### Add to Claude Code
+
+If installed via pip/uvx:
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "uvx",
+      "args": ["screaming-frog-mcp"],
+      "env": {
+        "SF_CLI_PATH": "/path/to/ScreamingFrogSEOSpiderLauncher"
+      }
+    }
+  }
+}
+```
+
+If cloned from source:
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "/path/to/screaming-frog-mcp/.venv/bin/python",
+      "args": ["/path/to/screaming-frog-mcp/sf_mcp.py"]
+    }
+  }
+}
+```
+
+### Add to Claude Desktop
+
+Add to your Claude Desktop config (`claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "uvx",
+      "args": ["screaming-frog-mcp"],
+      "env": {
+        "SF_CLI_PATH": "/path/to/ScreamingFrogSEOSpiderLauncher"
+      }
+    }
+  }
+}
+```
+
+## Available Tools
+
+| Tool | Description |
+|------|-------------|
+| `sf_check` | Verify Screaming Frog is installed, check version and license status |
+| `crawl_site` | Start a headless background crawl (see note below) |
+| `crawl_status` | Check progress of a running crawl |
+| `list_crawls` | List all saved crawls with their Database IDs |
+| `export_crawl` | Export crawl data as CSV files (many export options available) |
+| `read_crawl_data` | Read exported CSV data with pagination and filtering |
+| `delete_crawl` | Permanently delete a crawl from the database |
+| `storage_summary` | Show disk usage of SF's crawl storage |
+
+## Usage Examples
+
+### Check installation
+
+> "Is Screaming Frog installed and licensed?"
+
+The assistant will call `sf_check` and report version/license info.
+
+### Work with existing crawls (recommended flow)
+
+For most use cases, **crawl in the Screaming Frog GUI** where you have full control over configuration, JavaScript rendering, crawl scope, custom extraction, etc. Then close the GUI and use the MCP to analyze the results:
+
+After you've crawled a site in the Screaming Frog GUI and closed it:
+
+> "List my saved crawls"
+> "Export the crawl for example.com"
+> "Show me all pages with missing meta descriptions"
+> "What are the 404 pages?"
+
+### Crawl a site via MCP (optional)
+
+> "Crawl https://example.com with a max of 100 URLs"
+
+The `crawl_site` tool can kick off headless crawls via CLI. This is useful for quick re-crawls or automated workflows, but note the limitations compared to the GUI:
+- Uses default crawl settings (no custom extraction, JavaScript rendering config, etc.)
+- You can pass a `.seospiderconfig` file to customize settings, but the GUI is easier for complex setups
+- The crawl must finish and save before you can export data
+
+### Export options
+
+The server supports all of Screaming Frog's export tabs, bulk exports, and reports. Ask the assistant to read the `screaming-frog://export-reference` resource for the full list, or specify them directly:
+
+```
+export_tabs: "Internal:All,Response Codes:All,Page Titles:All"
+bulk_export: "All Inlinks,All Outlinks"
+save_report: "Crawl Overview"
+```
+
+## Temp file cleanup
+
+Exported CSVs are stored in `~/.cache/sf-mcp/exports/` and are automatically cleaned up after 1 hour.
+
+## Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| "GUI is already running" error | Quit the Screaming Frog application, then retry |
+| Empty CSV exports (headers only, 0 data rows) | The GUI likely has the database locked — close it and re-export |
+| CLI not found | Check that `SF_CLI_PATH` in `.env` points to the correct executable |
+| Crawl not appearing in `list_crawls` | Make sure you saved the crawl in the GUI (File > Save) before closing |
+| Export times out | Large crawls may need more time — try exporting fewer tabs |
+
+## License
+
+MIT
+
+<!-- mcp-name: io.github.bzsasson/screaming-frog-mcp -->

screaming_frog_mcp-0.1.0/README.md

@@ -0,0 +1,182 @@
+# Screaming Frog SEO Spider MCP Server
+
+An MCP (Model Context Protocol) server that gives Claude (or any MCP-compatible client) programmatic access to [Screaming Frog SEO Spider](https://www.screamingfrog.co.uk/seo-spider/) — crawl websites, export crawl data, and manage your crawl storage, all from your AI assistant.
+
+## Prerequisites
+
+1. **Screaming Frog SEO Spider** installed on your machine (tested with v23.x, should work with v16+).
+   Download from: https://www.screamingfrog.co.uk/seo-spider/
+
+2. **A valid Screaming Frog license.** The free version has a 500-URL crawl limit. Most MCP features (headless CLI, saving/loading crawls, exports) require a paid license.
+
+3. **Python 3.10+**
+
+## Important: How the Workflow Works
+
+Screaming Frog uses an internal database that can only be accessed by one process at a time. This means:
+
+> **You must close the Screaming Frog GUI before the MCP server can access crawl data.**
+
+The typical workflow is:
+
+1. **Run your crawl** — either through the SF GUI (with all your custom settings, filters, etc.) or via the MCP `crawl_site` tool.
+2. **Close the Screaming Frog GUI** — the GUI locks the crawl database. The MCP server's headless CLI cannot read or export data while the GUI is running.
+3. **Use the MCP tools** — once the GUI is closed, you can list crawls, export data, read CSVs, and more through your AI assistant.
+
+If you forget to close the GUI, the server will detect it and show a clear error message telling you to quit SF first.
+
+## Setup
+
+### Option A: Install from PyPI (recommended)
+
+```bash
+pip install screaming-frog-mcp
+```
+
+Or run directly with `uvx` (no install needed):
+
+```bash
+uvx screaming-frog-mcp
+```
+
+### Option B: Clone and install from source
+
+```bash
+git clone https://github.com/bzsasson/screaming-frog-mcp.git
+cd screaming-frog-mcp
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Configure the CLI path
+
+The default Screaming Frog CLI path works for macOS. If you're on Linux or Windows, set the `SF_CLI_PATH` environment variable:
+
+| OS      | Default Path |
+|---------|-------------|
+| macOS   | `/Applications/Screaming Frog SEO Spider.app/Contents/MacOS/ScreamingFrogSEOSpiderLauncher` |
+| Linux   | `/usr/bin/screamingfrogseospider` |
+| Windows | `C:\Program Files (x86)\Screaming Frog SEO Spider\ScreamingFrogSEOSpiderCli.exe` |
+
+If you cloned the repo, copy `.env.example` to `.env` and edit it.
+
+### Add to Claude Code
+
+If installed via pip/uvx:
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "uvx",
+      "args": ["screaming-frog-mcp"],
+      "env": {
+        "SF_CLI_PATH": "/path/to/ScreamingFrogSEOSpiderLauncher"
+      }
+    }
+  }
+}
+```
+
+If cloned from source:
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "/path/to/screaming-frog-mcp/.venv/bin/python",
+      "args": ["/path/to/screaming-frog-mcp/sf_mcp.py"]
+    }
+  }
+}
+```
+
+### Add to Claude Desktop
+
+Add to your Claude Desktop config (`claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "screaming-frog": {
+      "command": "uvx",
+      "args": ["screaming-frog-mcp"],
+      "env": {
+        "SF_CLI_PATH": "/path/to/ScreamingFrogSEOSpiderLauncher"
+      }
+    }
+  }
+}
+```
+
+## Available Tools
+
+| Tool | Description |
+|------|-------------|
+| `sf_check` | Verify Screaming Frog is installed, check version and license status |
+| `crawl_site` | Start a headless background crawl (see note below) |
+| `crawl_status` | Check progress of a running crawl |
+| `list_crawls` | List all saved crawls with their Database IDs |
+| `export_crawl` | Export crawl data as CSV files (many export options available) |
+| `read_crawl_data` | Read exported CSV data with pagination and filtering |
+| `delete_crawl` | Permanently delete a crawl from the database |
+| `storage_summary` | Show disk usage of SF's crawl storage |
+
+## Usage Examples
+
+### Check installation
+
+> "Is Screaming Frog installed and licensed?"
+
+The assistant will call `sf_check` and report version/license info.
+
+### Work with existing crawls (recommended flow)
+
+For most use cases, **crawl in the Screaming Frog GUI** where you have full control over configuration, JavaScript rendering, crawl scope, custom extraction, etc. Then close the GUI and use the MCP to analyze the results:
+
+After you've crawled a site in the Screaming Frog GUI and closed it:
+
+> "List my saved crawls"
+> "Export the crawl for example.com"
+> "Show me all pages with missing meta descriptions"
+> "What are the 404 pages?"
+
+### Crawl a site via MCP (optional)
+
+> "Crawl https://example.com with a max of 100 URLs"
+
+The `crawl_site` tool can kick off headless crawls via CLI. This is useful for quick re-crawls or automated workflows, but note the limitations compared to the GUI:
+- Uses default crawl settings (no custom extraction, JavaScript rendering config, etc.)
+- You can pass a `.seospiderconfig` file to customize settings, but the GUI is easier for complex setups
+- The crawl must finish and save before you can export data
+
+### Export options
+
+The server supports all of Screaming Frog's export tabs, bulk exports, and reports. Ask the assistant to read the `screaming-frog://export-reference` resource for the full list, or specify them directly:
+
+```
+export_tabs: "Internal:All,Response Codes:All,Page Titles:All"
+bulk_export: "All Inlinks,All Outlinks"
+save_report: "Crawl Overview"
+```
+
+## Temp file cleanup
+
+Exported CSVs are stored in `~/.cache/sf-mcp/exports/` and are automatically cleaned up after 1 hour.
+
+## Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| "GUI is already running" error | Quit the Screaming Frog application, then retry |
+| Empty CSV exports (headers only, 0 data rows) | The GUI likely has the database locked — close it and re-export |
+| CLI not found | Check that `SF_CLI_PATH` in `.env` points to the correct executable |
+| Crawl not appearing in `list_crawls` | Make sure you saved the crawl in the GUI (File > Save) before closing |
+| Export times out | Large crawls may need more time — try exporting fewer tabs |
+
+## License
+
+MIT
+
+<!-- mcp-name: io.github.bzsasson/screaming-frog-mcp -->

screaming_frog_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "screaming-frog-mcp"
+version = "0.1.0"
+description = "MCP server for Screaming Frog SEO Spider — crawl sites, export data, and manage crawl storage via AI assistants"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "Boaz Sasson" }]
+keywords = ["mcp", "seo", "screaming-frog", "crawl", "model-context-protocol"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP :: Site Management",
+]
+dependencies = [
+    "mcp>=1.26.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/bzsasson/screaming-frog-mcp"
+Repository = "https://github.com/bzsasson/screaming-frog-mcp"
+Issues = "https://github.com/bzsasson/screaming-frog-mcp/issues"
+
+[project.scripts]
+screaming-frog-mcp = "screaming_frog_mcp:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/screaming_frog_mcp"]

screaming_frog_mcp-0.1.0/requirements.txt

@@ -0,0 +1,2 @@
+mcp>=1.26.0
+python-dotenv>=1.0.0

screaming_frog_mcp-0.1.0/server.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.bzsasson/screaming-frog-mcp",
+  "title": "Screaming Frog SEO Spider MCP Server",
+  "description": "Crawl websites, export SEO data, and manage crawls via Screaming Frog SEO Spider.",
+  "version": "0.1.0",
+  "repository": {
+    "url": "https://github.com/bzsasson/screaming-frog-mcp",
+    "source": "github"
+  },
+  "packages": [
+    {
+      "registryType": "pypi",
+      "registryBaseUrl": "https://pypi.org",
+      "identifier": "screaming-frog-mcp",
+      "version": "0.1.0",
+      "transport": {
+        "type": "stdio"
+      },
+      "environmentVariables": [
+        {
+          "name": "SF_CLI_PATH",
+          "description": "Path to the Screaming Frog SEO Spider CLI executable. Defaults to the standard macOS location.",
+          "isRequired": false,
+          "isSecret": false
+        }
+      ]
+    }
+  ]
+}