searchfetch 3.3.2__tar.gz → 3.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searchfetch-3.3.2/searchfetch.egg-info → searchfetch-3.3.4}/PKG-INFO +34 -15
- {searchfetch-3.3.2 → searchfetch-3.3.4}/README.md +27 -11
- {searchfetch-3.3.2 → searchfetch-3.3.4}/pyproject.toml +17 -12
- {searchfetch-3.3.2 → searchfetch-3.3.4/searchfetch.egg-info}/PKG-INFO +34 -15
- {searchfetch-3.3.2 → searchfetch-3.3.4}/searchfetch.egg-info/SOURCES.txt +2 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/searchfetch.egg-info/entry_points.txt +0 -1
- searchfetch-3.3.4/searchfetch.egg-info/requires.txt +11 -0
- searchfetch-3.3.4/searchfetch.egg-info/top_level.txt +3 -0
- searchfetch-3.3.4/selectors_utils.py +31 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/server.py +304 -291
- searchfetch-3.3.4/templates/__init__.py +1 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/crates-package.json +1 -7
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/devto.json +1 -3
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/duckduckgo-search.json +3 -17
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/github-issue.json +1 -7
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/github-repo.json +2 -10
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/google-search.json +4 -18
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/javadoc.json +1 -3
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/mdn-web-docs.json +1 -3
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/npm-package.json +2 -10
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/pypi-package.json +1 -7
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/reddit.json +1 -3
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/wikipedia.json +1 -3
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/youtube.json +1 -3
- searchfetch-3.3.2/searchfetch.egg-info/requires.txt +0 -8
- searchfetch-3.3.2/searchfetch.egg-info/top_level.txt +0 -1
- {searchfetch-3.3.2 → searchfetch-3.3.4}/LICENSE +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/MANIFEST.in +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/searchfetch.egg-info/dependency_links.txt +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/setup.cfg +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/docker-hub.json +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/docs-page.json +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/docs-rs.json +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/gitlab.json +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/go-pkg.json +0 -0
- {searchfetch-3.3.2 → searchfetch-3.3.4}/templates/raw.json +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: searchfetch
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.4
|
|
4
4
|
Summary: A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching.
|
|
5
5
|
Author: Max
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,25 +10,28 @@ Keywords: mcp,search,fetch,llm,ai,agent,stealth
|
|
|
10
10
|
Requires-Python: >=3.10
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: mcp>=2.
|
|
14
|
-
Requires-Dist: cloakbrowser>=0.5.
|
|
13
|
+
Requires-Dist: mcp>=2.2.0
|
|
14
|
+
Requires-Dist: cloakbrowser>=0.5.10
|
|
15
15
|
Requires-Dist: beautifulsoup4>=4.15.0
|
|
16
16
|
Requires-Dist: markdownify>=1.2.3
|
|
17
|
+
Requires-Dist: playwright>=1.63.0
|
|
18
|
+
Requires-Dist: pydantic>=2.13.5
|
|
19
|
+
Requires-Dist: soupsieve>=2.9.2
|
|
17
20
|
Provides-Extra: dev
|
|
18
21
|
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
19
|
-
Requires-Dist: ruff>=0.16.
|
|
22
|
+
Requires-Dist: ruff>=0.16.7; extra == "dev"
|
|
20
23
|
Dynamic: license-file
|
|
21
24
|
|
|
22
25
|
# SearchFetch (MCP Server)
|
|
23
26
|
|
|
24
|
-
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a
|
|
27
|
+
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a browser to render pages and converts their content into Markdown.
|
|
25
28
|
|
|
26
29
|
## Features
|
|
27
30
|
|
|
28
|
-
- **
|
|
29
|
-
- **Fault
|
|
31
|
+
- **Browser rendering:** CloakBrowser runs Chromium with humanized interactions. Sites can still require authentication or present challenges.
|
|
32
|
+
- **Fault tolerance:** Reconnects disconnected browsers, retries network failures and HTTP 429 once, and blocks template-selected resource types. HTTP errors are reported before waiting for rendering.
|
|
30
33
|
- **Token-Optimized Output:** Strips base64 images, SVGs, scripts, and inline styles from the DOM _before_ Markdown conversion.
|
|
31
|
-
- **Dual
|
|
34
|
+
- **Dual runtime:** Run through Python (`uvx`, Python 3.10+) or Node.js (`npx`, Node 24+). First use downloads dependencies and a browser binary.
|
|
32
35
|
- **Template-Driven Extraction:** Structured extraction via shared JSON templates (GitHub, npm, PyPI, crates.io, docs pages, Docker Hub, and more). Supports custom inline templates.
|
|
33
36
|
|
|
34
37
|
---
|
|
@@ -84,7 +87,7 @@ Search the web through the template pipeline. DuckDuckGo and Google are built-in
|
|
|
84
87
|
| ------------- | ------------ | -------------- | ---------------------------------------------------------------------------------------------- |
|
|
85
88
|
| `query` | string | _required_ | The search query string. |
|
|
86
89
|
| `engine` | string | `"duckduckgo"` | `"duckduckgo"`, `"google"`, or a custom search template name. |
|
|
87
|
-
| `max_results` | number | `10` |
|
|
90
|
+
| `max_results` | number | `10` | Positive integer limit on extracted results. |
|
|
88
91
|
| `region` | string/null | `null` | Region/language code (e.g. `"us-en"`, `"de-de"`). DDG maps directly; Google maps to `gl`/`hl`. |
|
|
89
92
|
| `safe_search` | boolean/null | `null` | Enable safe search. `null` uses the template default. |
|
|
90
93
|
| `block_media` | boolean | `true` | Block images, media, and fonts at the network layer. |
|
|
@@ -97,11 +100,11 @@ Fetch a page with the stealth browser and extract structured Markdown using a te
|
|
|
97
100
|
| ------------- | ------- | ---------- | ----------------------------------------------------- |
|
|
98
101
|
| `url` | string | _required_ | Full URL (must start with `http`/`https`). |
|
|
99
102
|
| `template` | string | `"auto"` | `"auto"`, a built-in name, or inline JSON template. |
|
|
100
|
-
| `start_index` | number | `0` |
|
|
101
|
-
| `max_length` | number | `10000` |
|
|
103
|
+
| `start_index` | number | `0` | Nonnegative integer offset in Unicode code points. |
|
|
104
|
+
| `max_length` | number | `10000` | Positive integer limit in Unicode code points. |
|
|
102
105
|
| `block_media` | boolean | `true` | Block images, videos, and fonts at the network layer. |
|
|
103
106
|
|
|
104
|
-
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats;
|
|
107
|
+
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats; child fields within a section; repeated sections; URL-decoding transforms; per-template cookies; and per-template resource blocking.
|
|
105
108
|
|
|
106
109
|
Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
|
|
107
110
|
|
|
@@ -118,8 +121,8 @@ Built-in templates live in `templates/*.json` and are shared by the Node.js and
|
|
|
118
121
|
|
|
119
122
|
```bash
|
|
120
123
|
# Install dependencies
|
|
121
|
-
npm
|
|
122
|
-
|
|
124
|
+
npm ci
|
|
125
|
+
uv sync --locked --extra dev
|
|
123
126
|
|
|
124
127
|
# Run tests
|
|
125
128
|
npm test # runs all tests (JS + Python)
|
|
@@ -144,4 +147,20 @@ npm run inspector-py # test with MCP Inspector (Python)
|
|
|
144
147
|
|
|
145
148
|
## Architecture
|
|
146
149
|
|
|
147
|
-
|
|
150
|
+
Both MCP servers expose `websearch` and `webfetch` over standard input/output:
|
|
151
|
+
|
|
152
|
+
1. Validate tool inputs and resolve a built-in or inline JSON template.
|
|
153
|
+
2. Reuse one browser, creating an isolated browser context for each fetch attempt.
|
|
154
|
+
3. Apply template cookies and resource blocking, navigate, check HTTP status, and allow up to five seconds for network activity to settle.
|
|
155
|
+
4. Remove configured elements, extract section fields, apply transforms, and compose Markdown. Page templates may first try a raw Markdown source URL.
|
|
156
|
+
5. Paginate page output using Unicode code points. Search requests can fall back from Google to DuckDuckGo HTML and then Lite; output names any fallback engine.
|
|
157
|
+
|
|
158
|
+
`index.js` and `server.py` contain the runtime-specific browser and MCP integration. `lib/` and `selectors_utils.py` hold focused formatting and selector helpers. Both runtimes read the same `templates/*.json`; Python wheels bundle these as `searchfetch_templates` resources. Shared fixtures in `tests/fixtures/` exercise extraction behavior in both runtimes.
|
|
159
|
+
|
|
160
|
+
Selectors separated by top-level commas are ordered fallbacks. Commas inside CSS functions or attributes are preserved; an empty fallback selects the current element. Child extraction searches descendants and enclosing elements, without borrowing fields from neighboring results. Missing required fields and malformed selectors report errors.
|
|
161
|
+
|
|
162
|
+
## Verification and limits
|
|
163
|
+
|
|
164
|
+
`npm test`, `npm run lint`, and `npm run format:check` check both runtimes. `npm run e2e` runs real browser requests against local HTTP fixtures, exercises every built-in template, and checks the installed npm executable and Python wheel. Public search-engine availability and changing third-party page layouts require separate live checks. A page that continues rendering beyond the bounded wait may return partial content.
|
|
165
|
+
|
|
166
|
+
`package-lock.json` and `uv.lock` record dependency resolution.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# SearchFetch (MCP Server)
|
|
2
2
|
|
|
3
|
-
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a
|
|
3
|
+
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a browser to render pages and converts their content into Markdown.
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
|
7
|
-
- **
|
|
8
|
-
- **Fault
|
|
7
|
+
- **Browser rendering:** CloakBrowser runs Chromium with humanized interactions. Sites can still require authentication or present challenges.
|
|
8
|
+
- **Fault tolerance:** Reconnects disconnected browsers, retries network failures and HTTP 429 once, and blocks template-selected resource types. HTTP errors are reported before waiting for rendering.
|
|
9
9
|
- **Token-Optimized Output:** Strips base64 images, SVGs, scripts, and inline styles from the DOM _before_ Markdown conversion.
|
|
10
|
-
- **Dual
|
|
10
|
+
- **Dual runtime:** Run through Python (`uvx`, Python 3.10+) or Node.js (`npx`, Node 24+). First use downloads dependencies and a browser binary.
|
|
11
11
|
- **Template-Driven Extraction:** Structured extraction via shared JSON templates (GitHub, npm, PyPI, crates.io, docs pages, Docker Hub, and more). Supports custom inline templates.
|
|
12
12
|
|
|
13
13
|
---
|
|
@@ -63,7 +63,7 @@ Search the web through the template pipeline. DuckDuckGo and Google are built-in
|
|
|
63
63
|
| ------------- | ------------ | -------------- | ---------------------------------------------------------------------------------------------- |
|
|
64
64
|
| `query` | string | _required_ | The search query string. |
|
|
65
65
|
| `engine` | string | `"duckduckgo"` | `"duckduckgo"`, `"google"`, or a custom search template name. |
|
|
66
|
-
| `max_results` | number | `10` |
|
|
66
|
+
| `max_results` | number | `10` | Positive integer limit on extracted results. |
|
|
67
67
|
| `region` | string/null | `null` | Region/language code (e.g. `"us-en"`, `"de-de"`). DDG maps directly; Google maps to `gl`/`hl`. |
|
|
68
68
|
| `safe_search` | boolean/null | `null` | Enable safe search. `null` uses the template default. |
|
|
69
69
|
| `block_media` | boolean | `true` | Block images, media, and fonts at the network layer. |
|
|
@@ -76,11 +76,11 @@ Fetch a page with the stealth browser and extract structured Markdown using a te
|
|
|
76
76
|
| ------------- | ------- | ---------- | ----------------------------------------------------- |
|
|
77
77
|
| `url` | string | _required_ | Full URL (must start with `http`/`https`). |
|
|
78
78
|
| `template` | string | `"auto"` | `"auto"`, a built-in name, or inline JSON template. |
|
|
79
|
-
| `start_index` | number | `0` |
|
|
80
|
-
| `max_length` | number | `10000` |
|
|
79
|
+
| `start_index` | number | `0` | Nonnegative integer offset in Unicode code points. |
|
|
80
|
+
| `max_length` | number | `10000` | Positive integer limit in Unicode code points. |
|
|
81
81
|
| `block_media` | boolean | `true` | Block images, videos, and fonts at the network layer. |
|
|
82
82
|
|
|
83
|
-
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats;
|
|
83
|
+
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats; child fields within a section; repeated sections; URL-decoding transforms; per-template cookies; and per-template resource blocking.
|
|
84
84
|
|
|
85
85
|
Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
|
|
86
86
|
|
|
@@ -97,8 +97,8 @@ Built-in templates live in `templates/*.json` and are shared by the Node.js and
|
|
|
97
97
|
|
|
98
98
|
```bash
|
|
99
99
|
# Install dependencies
|
|
100
|
-
npm
|
|
101
|
-
|
|
100
|
+
npm ci
|
|
101
|
+
uv sync --locked --extra dev
|
|
102
102
|
|
|
103
103
|
# Run tests
|
|
104
104
|
npm test # runs all tests (JS + Python)
|
|
@@ -123,4 +123,20 @@ npm run inspector-py # test with MCP Inspector (Python)
|
|
|
123
123
|
|
|
124
124
|
## Architecture
|
|
125
125
|
|
|
126
|
-
|
|
126
|
+
Both MCP servers expose `websearch` and `webfetch` over standard input/output:
|
|
127
|
+
|
|
128
|
+
1. Validate tool inputs and resolve a built-in or inline JSON template.
|
|
129
|
+
2. Reuse one browser, creating an isolated browser context for each fetch attempt.
|
|
130
|
+
3. Apply template cookies and resource blocking, navigate, check HTTP status, and allow up to five seconds for network activity to settle.
|
|
131
|
+
4. Remove configured elements, extract section fields, apply transforms, and compose Markdown. Page templates may first try a raw Markdown source URL.
|
|
132
|
+
5. Paginate page output using Unicode code points. Search requests can fall back from Google to DuckDuckGo HTML and then Lite; output names any fallback engine.
|
|
133
|
+
|
|
134
|
+
`index.js` and `server.py` contain the runtime-specific browser and MCP integration. `lib/` and `selectors_utils.py` hold focused formatting and selector helpers. Both runtimes read the same `templates/*.json`; Python wheels bundle these as `searchfetch_templates` resources. Shared fixtures in `tests/fixtures/` exercise extraction behavior in both runtimes.
|
|
135
|
+
|
|
136
|
+
Selectors separated by top-level commas are ordered fallbacks. Commas inside CSS functions or attributes are preserved; an empty fallback selects the current element. Child extraction searches descendants and enclosing elements, without borrowing fields from neighboring results. Missing required fields and malformed selectors report errors.
|
|
137
|
+
|
|
138
|
+
## Verification and limits
|
|
139
|
+
|
|
140
|
+
`npm test`, `npm run lint`, and `npm run format:check` check both runtimes. `npm run e2e` runs real browser requests against local HTTP fixtures, exercises every built-in template, and checks the installed npm executable and Python wheel. Public search-engine availability and changing third-party page layouts require separate live checks. A page that continues rendering beyond the bounded wait may return partial content.
|
|
141
|
+
|
|
142
|
+
`package-lock.json` and `uv.lock` record dependency resolution.
|
|
@@ -1,29 +1,32 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=84.0.0"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "searchfetch"
|
|
7
|
-
version = "3.3.
|
|
7
|
+
version = "3.3.4"
|
|
8
8
|
description = "A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
requires-python = ">=3.10"
|
|
12
|
-
authors =[
|
|
12
|
+
authors = [
|
|
13
13
|
{ name = "Max" }
|
|
14
14
|
]
|
|
15
|
-
keywords =["mcp", "search", "fetch", "llm", "ai", "agent", "stealth"]
|
|
16
|
-
dependencies =[
|
|
17
|
-
"mcp>=2.
|
|
18
|
-
"cloakbrowser>=0.5.
|
|
15
|
+
keywords = ["mcp", "search", "fetch", "llm", "ai", "agent", "stealth"]
|
|
16
|
+
dependencies = [
|
|
17
|
+
"mcp>=2.2.0",
|
|
18
|
+
"cloakbrowser>=0.5.10",
|
|
19
19
|
"beautifulsoup4>=4.15.0",
|
|
20
20
|
"markdownify>=1.2.3",
|
|
21
|
+
"playwright>=1.63.0",
|
|
22
|
+
"pydantic>=2.13.5",
|
|
23
|
+
"soupsieve>=2.9.2",
|
|
21
24
|
]
|
|
22
25
|
|
|
23
26
|
[project.optional-dependencies]
|
|
24
27
|
dev = [
|
|
25
28
|
"pytest>=9.1.1",
|
|
26
|
-
"ruff>=0.16.
|
|
29
|
+
"ruff>=0.16.7",
|
|
27
30
|
]
|
|
28
31
|
|
|
29
32
|
[project.urls]
|
|
@@ -31,14 +34,16 @@ Homepage = "https://github.com/maxylev/searchfetch"
|
|
|
31
34
|
Repository = "https://github.com/maxylev/searchfetch"
|
|
32
35
|
|
|
33
36
|
[project.scripts]
|
|
34
|
-
mcp-searchfetch = "server:main"
|
|
35
37
|
searchfetch = "server:main"
|
|
36
38
|
|
|
37
39
|
[tool.setuptools]
|
|
38
|
-
py-modules =["server"]
|
|
40
|
+
py-modules = ["server", "selectors_utils"]
|
|
39
41
|
|
|
40
|
-
[
|
|
41
|
-
|
|
42
|
+
packages = ["searchfetch_templates"]
|
|
43
|
+
package-dir = { searchfetch_templates = "templates" }
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.package-data]
|
|
46
|
+
searchfetch_templates = ["*.json"]
|
|
42
47
|
|
|
43
48
|
[tool.ruff]
|
|
44
49
|
line-length = 100
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: searchfetch
|
|
3
|
-
Version: 3.3.
|
|
3
|
+
Version: 3.3.4
|
|
4
4
|
Summary: A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching.
|
|
5
5
|
Author: Max
|
|
6
6
|
License-Expression: MIT
|
|
@@ -10,25 +10,28 @@ Keywords: mcp,search,fetch,llm,ai,agent,stealth
|
|
|
10
10
|
Requires-Python: >=3.10
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: mcp>=2.
|
|
14
|
-
Requires-Dist: cloakbrowser>=0.5.
|
|
13
|
+
Requires-Dist: mcp>=2.2.0
|
|
14
|
+
Requires-Dist: cloakbrowser>=0.5.10
|
|
15
15
|
Requires-Dist: beautifulsoup4>=4.15.0
|
|
16
16
|
Requires-Dist: markdownify>=1.2.3
|
|
17
|
+
Requires-Dist: playwright>=1.63.0
|
|
18
|
+
Requires-Dist: pydantic>=2.13.5
|
|
19
|
+
Requires-Dist: soupsieve>=2.9.2
|
|
17
20
|
Provides-Extra: dev
|
|
18
21
|
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
19
|
-
Requires-Dist: ruff>=0.16.
|
|
22
|
+
Requires-Dist: ruff>=0.16.7; extra == "dev"
|
|
20
23
|
Dynamic: license-file
|
|
21
24
|
|
|
22
25
|
# SearchFetch (MCP Server)
|
|
23
26
|
|
|
24
|
-
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a
|
|
27
|
+
A fault-tolerant, stealth-enabled Model Context Protocol (MCP) server for web searching and content fetching. Built for AI Agents (Cursor, Claude Code, OpenCode), it uses a browser to render pages and converts their content into Markdown.
|
|
25
28
|
|
|
26
29
|
## Features
|
|
27
30
|
|
|
28
|
-
- **
|
|
29
|
-
- **Fault
|
|
31
|
+
- **Browser rendering:** CloakBrowser runs Chromium with humanized interactions. Sites can still require authentication or present challenges.
|
|
32
|
+
- **Fault tolerance:** Reconnects disconnected browsers, retries network failures and HTTP 429 once, and blocks template-selected resource types. HTTP errors are reported before waiting for rendering.
|
|
30
33
|
- **Token-Optimized Output:** Strips base64 images, SVGs, scripts, and inline styles from the DOM _before_ Markdown conversion.
|
|
31
|
-
- **Dual
|
|
34
|
+
- **Dual runtime:** Run through Python (`uvx`, Python 3.10+) or Node.js (`npx`, Node 24+). First use downloads dependencies and a browser binary.
|
|
32
35
|
- **Template-Driven Extraction:** Structured extraction via shared JSON templates (GitHub, npm, PyPI, crates.io, docs pages, Docker Hub, and more). Supports custom inline templates.
|
|
33
36
|
|
|
34
37
|
---
|
|
@@ -84,7 +87,7 @@ Search the web through the template pipeline. DuckDuckGo and Google are built-in
|
|
|
84
87
|
| ------------- | ------------ | -------------- | ---------------------------------------------------------------------------------------------- |
|
|
85
88
|
| `query` | string | _required_ | The search query string. |
|
|
86
89
|
| `engine` | string | `"duckduckgo"` | `"duckduckgo"`, `"google"`, or a custom search template name. |
|
|
87
|
-
| `max_results` | number | `10` |
|
|
90
|
+
| `max_results` | number | `10` | Positive integer limit on extracted results. |
|
|
88
91
|
| `region` | string/null | `null` | Region/language code (e.g. `"us-en"`, `"de-de"`). DDG maps directly; Google maps to `gl`/`hl`. |
|
|
89
92
|
| `safe_search` | boolean/null | `null` | Enable safe search. `null` uses the template default. |
|
|
90
93
|
| `block_media` | boolean | `true` | Block images, media, and fonts at the network layer. |
|
|
@@ -97,11 +100,11 @@ Fetch a page with the stealth browser and extract structured Markdown using a te
|
|
|
97
100
|
| ------------- | ------- | ---------- | ----------------------------------------------------- |
|
|
98
101
|
| `url` | string | _required_ | Full URL (must start with `http`/`https`). |
|
|
99
102
|
| `template` | string | `"auto"` | `"auto"`, a built-in name, or inline JSON template. |
|
|
100
|
-
| `start_index` | number | `0` |
|
|
101
|
-
| `max_length` | number | `10000` |
|
|
103
|
+
| `start_index` | number | `0` | Nonnegative integer offset in Unicode code points. |
|
|
104
|
+
| `max_length` | number | `10000` | Positive integer limit in Unicode code points. |
|
|
102
105
|
| `block_media` | boolean | `true` | Block images, videos, and fonts at the network layer. |
|
|
103
106
|
|
|
104
|
-
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats;
|
|
107
|
+
Template extraction supports `text`, `markdown`, `attribute`, and `html` formats; child fields within a section; repeated sections; URL-decoding transforms; per-template cookies; and per-template resource blocking.
|
|
105
108
|
|
|
106
109
|
Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
|
|
107
110
|
|
|
@@ -118,8 +121,8 @@ Built-in templates live in `templates/*.json` and are shared by the Node.js and
|
|
|
118
121
|
|
|
119
122
|
```bash
|
|
120
123
|
# Install dependencies
|
|
121
|
-
npm
|
|
122
|
-
|
|
124
|
+
npm ci
|
|
125
|
+
uv sync --locked --extra dev
|
|
123
126
|
|
|
124
127
|
# Run tests
|
|
125
128
|
npm test # runs all tests (JS + Python)
|
|
@@ -144,4 +147,20 @@ npm run inspector-py # test with MCP Inspector (Python)
|
|
|
144
147
|
|
|
145
148
|
## Architecture
|
|
146
149
|
|
|
147
|
-
|
|
150
|
+
Both MCP servers expose `websearch` and `webfetch` over standard input/output:
|
|
151
|
+
|
|
152
|
+
1. Validate tool inputs and resolve a built-in or inline JSON template.
|
|
153
|
+
2. Reuse one browser, creating an isolated browser context for each fetch attempt.
|
|
154
|
+
3. Apply template cookies and resource blocking, navigate, check HTTP status, and allow up to five seconds for network activity to settle.
|
|
155
|
+
4. Remove configured elements, extract section fields, apply transforms, and compose Markdown. Page templates may first try a raw Markdown source URL.
|
|
156
|
+
5. Paginate page output using Unicode code points. Search requests can fall back from Google to DuckDuckGo HTML and then Lite; output names any fallback engine.
|
|
157
|
+
|
|
158
|
+
`index.js` and `server.py` contain the runtime-specific browser and MCP integration. `lib/` and `selectors_utils.py` hold focused formatting and selector helpers. Both runtimes read the same `templates/*.json`; Python wheels bundle these as `searchfetch_templates` resources. Shared fixtures in `tests/fixtures/` exercise extraction behavior in both runtimes.
|
|
159
|
+
|
|
160
|
+
Selectors separated by top-level commas are ordered fallbacks. Commas inside CSS functions or attributes are preserved; an empty fallback selects the current element. Child extraction searches descendants and enclosing elements, without borrowing fields from neighboring results. Missing required fields and malformed selectors report errors.
|
|
161
|
+
|
|
162
|
+
## Verification and limits
|
|
163
|
+
|
|
164
|
+
`npm test`, `npm run lint`, and `npm run format:check` check both runtimes. `npm run e2e` runs real browser requests against local HTTP fixtures, exercises every built-in template, and checks the installed npm executable and Python wheel. Public search-engine availability and changing third-party page layouts require separate live checks. A page that continues rendering beyond the bounded wait may return partial content.
|
|
165
|
+
|
|
166
|
+
`package-lock.json` and `uv.lock` record dependency resolution.
|
|
@@ -2,6 +2,7 @@ LICENSE
|
|
|
2
2
|
MANIFEST.in
|
|
3
3
|
README.md
|
|
4
4
|
pyproject.toml
|
|
5
|
+
selectors_utils.py
|
|
5
6
|
server.py
|
|
6
7
|
searchfetch.egg-info/PKG-INFO
|
|
7
8
|
searchfetch.egg-info/SOURCES.txt
|
|
@@ -9,6 +10,7 @@ searchfetch.egg-info/dependency_links.txt
|
|
|
9
10
|
searchfetch.egg-info/entry_points.txt
|
|
10
11
|
searchfetch.egg-info/requires.txt
|
|
11
12
|
searchfetch.egg-info/top_level.txt
|
|
13
|
+
templates/__init__.py
|
|
12
14
|
templates/crates-package.json
|
|
13
15
|
templates/devto.json
|
|
14
16
|
templates/docker-hub.json
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Parsing for ordered CSS selector fallbacks shared by extraction entry points."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def split_selectors(value: str = "") -> list[str]:
|
|
5
|
+
"""Preserve commas inside CSS functions, quoted attributes, and escapes."""
|
|
6
|
+
selectors = []
|
|
7
|
+
start = depth = 0
|
|
8
|
+
quote = None
|
|
9
|
+
escaped = False
|
|
10
|
+
for index, char in enumerate(value):
|
|
11
|
+
if escaped:
|
|
12
|
+
escaped = False
|
|
13
|
+
continue
|
|
14
|
+
if char == "\\":
|
|
15
|
+
escaped = True
|
|
16
|
+
continue
|
|
17
|
+
if quote:
|
|
18
|
+
if char == quote:
|
|
19
|
+
quote = None
|
|
20
|
+
continue
|
|
21
|
+
if char in ('"', "'"):
|
|
22
|
+
quote = char
|
|
23
|
+
elif char in "([":
|
|
24
|
+
depth += 1
|
|
25
|
+
elif char in ")]":
|
|
26
|
+
depth -= 1
|
|
27
|
+
elif char == "," and depth == 0:
|
|
28
|
+
selectors.append(value[start:index].strip())
|
|
29
|
+
start = index + 1
|
|
30
|
+
selectors.append(value[start:].strip())
|
|
31
|
+
return selectors
|