searchfetch 3.2.2__tar.gz → 3.3.0__tar.gz

{searchfetch-3.2.2/searchfetch.egg-info → searchfetch-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: searchfetch
-Version: 3.2.2
+Version: 3.3.0
 Summary: A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching.
 Author: Max
 License-Expression: MIT
@@ -105,6 +105,13 @@ Template extraction supports `text`, `markdown`, `attribute`, and `html` formats
 
 Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
 
+**Available page templates** (auto-detected by URL or selectable by name):
+`wikipedia`, `reddit`, `mdn-web-docs`, `gitlab`, `youtube`, `devto`, `go-pkg`, `javadoc`,
+`github-repo`, `github-issue`, `npm-package`, `pypi-package`, `crates-package`,
+`docker-hub`, `docs-rs`, `docs-page`
+
+**`raw`** — special template that applies minimal filtering and returns full body content as markdown. Use when you need the complete page without template-specific extraction.
+
 ---
 
 ## Local Development

{searchfetch-3.2.2 → searchfetch-3.3.0}/README.md

@@ -84,6 +84,13 @@ Template extraction supports `text`, `markdown`, `attribute`, and `html` formats
 
 Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
 
+**Available page templates** (auto-detected by URL or selectable by name):
+`wikipedia`, `reddit`, `mdn-web-docs`, `gitlab`, `youtube`, `devto`, `go-pkg`, `javadoc`,
+`github-repo`, `github-issue`, `npm-package`, `pypi-package`, `crates-package`,
+`docker-hub`, `docs-rs`, `docs-page`
+
+**`raw`** — special template that applies minimal filtering and returns full body content as markdown. Use when you need the complete page without template-specific extraction.
+
 ---
 
 ## Local Development

{searchfetch-3.2.2 → searchfetch-3.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "searchfetch"
-version = "3.2.2"
+version = "3.3.0"
 description = "A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching."
 readme = "README.md"
 license = "MIT"

{searchfetch-3.2.2 → searchfetch-3.3.0/searchfetch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: searchfetch
-Version: 3.2.2
+Version: 3.3.0
 Summary: A maximum fault-tolerant, stealth-enabled MCP server for web searching and fetching.
 Author: Max
 License-Expression: MIT
@@ -105,6 +105,13 @@ Template extraction supports `text`, `markdown`, `attribute`, and `html` formats
 
 Built-in templates live in `templates/*.json` and are shared by the Node.js and Python implementations.
 
+**Available page templates** (auto-detected by URL or selectable by name):
+`wikipedia`, `reddit`, `mdn-web-docs`, `gitlab`, `youtube`, `devto`, `go-pkg`, `javadoc`,
+`github-repo`, `github-issue`, `npm-package`, `pypi-package`, `crates-package`,
+`docker-hub`, `docs-rs`, `docs-page`
+
+**`raw`** — special template that applies minimal filtering and returns full body content as markdown. Use when you need the complete page without template-specific extraction.
+
 ---
 
 ## Local Development

{searchfetch-3.2.2 → searchfetch-3.3.0}/searchfetch.egg-info/SOURCES.txt

@@ -10,12 +10,21 @@ searchfetch.egg-info/entry_points.txt
 searchfetch.egg-info/requires.txt
 searchfetch.egg-info/top_level.txt
 templates/crates-package.json
+templates/devto.json
 templates/docker-hub.json
 templates/docs-page.json
 templates/docs-rs.json
 templates/duckduckgo-search.json
 templates/github-issue.json
 templates/github-repo.json
+templates/gitlab.json
+templates/go-pkg.json
 templates/google-search.json
+templates/javadoc.json
+templates/mdn-web-docs.json
 templates/npm-package.json
-templates/pypi-package.json
+templates/pypi-package.json
+templates/raw.json
+templates/reddit.json
+templates/wikipedia.json
+templates/youtube.json

{searchfetch-3.2.2 → searchfetch-3.3.0}/server.py

@@ -11,7 +11,7 @@ from markdownify import markdownify as md
 from mcp.server.fastmcp import FastMCP
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 
-__version__ = "3.2.2"
+__version__ = "3.3.0"
 
 mcp = FastMCP("searchfetch")
 
@@ -1108,14 +1108,25 @@ async def webfetch(
     block_media: bool = True,
 ) -> str:
     """
-    Fetch a webpage and extract structured information using templates.
+    Fetch and extract the main text content from any webpage. Fully executes
+    JavaScript to load React/SPAs and aggressively strips images/media
+    (including base64) to save context tokens.
 
     Args:
-        url: The full URL to fetch.
-        template: "auto", a built-in name, or inline JSON. Default: "auto".
+        url: The full URL of the webpage to fetch (must start with http/https).
+        template: Template to use: 'auto' (auto-detect from URL), a built-in
+                  page template name (see list below), 'raw' for
+                  minimal-filtering full-page output, or inline JSON.
+                  Default: 'auto'.
         start_index: Character offset for pagination. Default: 0.
-        max_length: Max characters to return. Default: 10000.
-        block_media: Block images/media/fonts. Default: True.
+        max_length: Maximum characters to return per request. Default is 10000.
+        block_media: Block images, videos, and fonts entirely at the network
+                     layer. Default is true.
+
+    Built-in page templates (use by name to force a specific extractor):
+      wikipedia, reddit, mdn-web-docs, gitlab, youtube, devto, go-pkg,
+      javadoc, github-repo, github-issue, npm-package, pypi-package,
+      crates-package, docker-hub, docs-rs, docs-page, raw
     """
     # 1. Resolve template
     matched_template, template_name = resolve_page_template(url, template)
@@ -1167,7 +1178,10 @@ async def webfetch(
 
 
 def main():
-    mcp.run(transport="stdio")
+    try:
+        mcp.run(transport="stdio")
+    except BrokenPipeError:
+        pass
 
 
 if __name__ == "__main__":

searchfetch-3.3.0/templates/devto.json

@@ -0,0 +1,45 @@
+{
+  "name": "devto",
+  "description": "DEV Community article — title, author, and content",
+  "order": 20,
+  "url_patterns": [
+    "^https?://dev\\.to/[^/]+/[^/]+"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "img",
+    "nav",
+    "footer",
+    ".sidebar",
+    ".crayons-layout__sidebar-left",
+    ".crayons-layout__sidebar-right",
+    ".crayons-article-actions",
+    ".reaction-button",
+    ".crayons-btn",
+    ".spec__tags",
+    ".article-engagement-bar",
+    ".comments-container"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "h1",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Author",
+      "selector": ".crayons-article__header__meta a, a.crayons-avatar",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Content",
+      "selector": "#article-body, .crayons-article__body, article.crayons-article",
+      "format": "markdown",
+      "required": true
+    }
+  ]
+}

searchfetch-3.3.0/templates/gitlab.json

@@ -0,0 +1,49 @@
+{
+  "name": "gitlab",
+  "description": "GitLab project — description, file listing, and README",
+  "order": 12,
+  "url_patterns": [
+    "^https?://gitlab\\.com/[^/]+/[^/]+/?$",
+    "^https?://gitlab\\.com/[^/]+/[^/]+/-/tree/[^/]+/?$"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    "header",
+    ".navbar-gitlab",
+    ".nav-sidebar",
+    ".issuable-list",
+    ".super-sidebar",
+    "aside"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "h1",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Description",
+      "selector": "meta[name='description']",
+      "format": "attribute",
+      "attribute": "content",
+      "required": false
+    },
+    {
+      "name": "Files",
+      "selector": "table[data-qa-selector='file_tree_table'], .tree-table",
+      "format": "markdown",
+      "required": false
+    },
+    {
+      "name": "README",
+      "selector": ".readme-holder .md, .file-content .md, .readme-holder",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

searchfetch-3.3.0/templates/go-pkg.json

@@ -0,0 +1,50 @@
+{
+  "name": "go-pkg",
+  "description": "Go package documentation on pkg.go.dev — name, overview, and docs",
+  "order": 22,
+  "url_patterns": [
+    "^https?://pkg\\.go\\.dev/[^/]+/?$",
+    "^https?://pkg\\.go\\.dev/[^/]+/[^/]+/?$",
+    "^https?://pkg\\.go\\.dev/[^/]+/[^/]+/[^/]+/?$"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    ".go-Header",
+    ".go-SiteHeader",
+    ".go-Footer",
+    ".UnitMeta",
+    ".UnitMeta-repo",
+    ".UnitFiles",
+    ".JumpDrawer",
+    ".go-Main-nav",
+    ".go-Main-headerSubmenu",
+    ".Site-header",
+    ".UnitHeader-version",
+    ".UnitHeader-breadcrumb",
+    ".UnitHeader-details"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "h1",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Overview",
+      "selector": ".Documentation-overview, .Overview",
+      "format": "markdown",
+      "required": false
+    },
+    {
+      "name": "Documentation",
+      "selector": ".Documentation-content, .Documentation, #documentation",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

searchfetch-3.3.0/templates/javadoc.json

@@ -0,0 +1,43 @@
+{
+  "name": "javadoc",
+  "description": "Java SE API documentation — class/method details",
+  "order": 8.5,
+  "url_patterns": [
+    "^https?://docs\\.oracle\\.com/.*/docs/api/.*"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    "header",
+    ".topNav",
+    ".bottomNav",
+    ".subNav",
+    ".navListSearch",
+    ".skipNav",
+    ".searchTag",
+    ".searchTagResult",
+    ".useSummary",
+    ".inheritedList",
+    ".details",
+    ".memberSummary",
+    ".colFirst",
+    ".colLast"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": ".header h2, h2.title, h2:first-of-type",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Content",
+      "selector": "[role='main'], .document, .rst-content, article, .content, .markdown-body, .prose, main, .contentContainer",
+      "format": "markdown",
+      "required": true
+    }
+  ]
+}

searchfetch-3.3.0/templates/mdn-web-docs.json

@@ -0,0 +1,39 @@
+{
+  "name": "mdn-web-docs",
+  "description": "MDN Web Docs article — title and main content",
+  "order": 15,
+  "url_patterns": [
+    "^https?://developer\\.mozilla\\.org/.*"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    "header",
+    ".sidebar",
+    ".toc",
+    ".interactive",
+    ".playground",
+    ".bc-data",
+    ".prev-next",
+    ".notecard",
+    ".language-menu",
+    ".breadcrumbs-container"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "h1",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Content",
+      "selector": "#content, article, main, .main-page-content",
+      "format": "markdown",
+      "required": true
+    }
+  ]
+}

searchfetch-3.3.0/templates/raw.json

@@ -0,0 +1,14 @@
+{
+  "name": "raw",
+  "description": "Raw page content — minimal filtering, full body as markdown (use when you need the complete page without template extraction)",
+  "order": 999,
+  "remove": [],
+  "sections": [
+    {
+      "name": "Content",
+      "selector": "body",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

searchfetch-3.3.0/templates/reddit.json

@@ -0,0 +1,65 @@
+{
+  "name": "reddit",
+  "description": "Reddit post — title, content, and comments",
+  "order": 13,
+  "url_patterns": [
+    "^https?://(?:www\\.|old\\.)?reddit\\.com/r/[^/]+/comments/[^/]+"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "img",
+    "nav",
+    "footer",
+    "header",
+    ".promoted-post",
+    "shreddit-ad-post",
+    ".side",
+    ".sidebar",
+    ".footer-parent",
+    "#header",
+    ".sr-header-area",
+    ".menuarea",
+    ".linkinfo",
+    ".panestack-title",
+    ".report-button",
+    "shreddit-join-button",
+    "#overlayScrollContainer",
+    "#right-sidebar-container",
+    "#left-sidebar-container"
+  ],
+  "sections": [
+    {
+      "name": "Subreddit",
+      "selector": "a[href*='/r/'], shreddit-post a[href*='/r/']",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Title",
+      "selector": "h1, shreddit-post [slot='title']",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Author",
+      "selector": "a[href*='/user/'], shreddit-post a[href*='/user/'], [slot='authorName']",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Link",
+      "selector": "a[slot='external-anchor'], a.outbound-link, a[data-testid='outbound-link']",
+      "format": "attribute",
+      "attribute": "href",
+      "required": false
+    },
+    {
+      "name": "Content",
+      "selector": "[slot='text-body'], .text-body, .expando, .usertext-body .md",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}

searchfetch-3.3.0/templates/wikipedia.json

@@ -0,0 +1,46 @@
+{
+  "name": "wikipedia",
+  "description": "Wikipedia article — title and main content",
+  "order": 11,
+  "url_patterns": [
+    "^https?://[a-z]{2,4}\\.wikipedia\\.org/wiki/[^:]+$"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "nav",
+    "footer",
+    ".mw-editsection",
+    ".reflist",
+    ".navbox",
+    ".sisterproject",
+    ".metadata",
+    ".sidebar",
+    ".infobox",
+    "#toc",
+    ".mw-jump-link",
+    ".printfooter",
+    ".catlinks",
+    ".mw-authority-control",
+    ".shortdescription",
+    ".mw-indicators",
+    ".page-actions",
+    ".mw-page-title-namespace",
+    ".mw-heading-subtitle"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "#firstHeading, h1",
+      "format": "text",
+      "required": true
+    },
+    {
+      "name": "Content",
+      "selector": "#mw-content-text .mw-parser-output",
+      "format": "markdown",
+      "required": true
+    }
+  ]
+}

searchfetch-3.3.0/templates/youtube.json

@@ -0,0 +1,44 @@
+{
+  "name": "youtube",
+  "description": "YouTube video — title, channel, description",
+  "order": 30,
+  "url_patterns": [
+    "^https?://(?:www\\.)?(?:youtube\\.com/watch\\?v=|youtu\\.be/).+"
+  ],
+  "remove": [
+    "script",
+    "style",
+    "svg",
+    "img",
+    "nav",
+    "header",
+    "ytd-guide-renderer",
+    "ytd-mini-guide-renderer",
+    "#comments",
+    "#related",
+    "ytd-comments",
+    "ytd-live-chat-frame",
+    ".ytp-ad-module"
+  ],
+  "sections": [
+    {
+      "name": "Title",
+      "selector": "meta[name='title']",
+      "format": "attribute",
+      "attribute": "content",
+      "required": true
+    },
+    {
+      "name": "Channel",
+      "selector": "#owner a, ytd-video-owner-renderer a, #channel-name a",
+      "format": "text",
+      "required": false
+    },
+    {
+      "name": "Description",
+      "selector": "#description-inline-expander, ytd-text-inline-expander, #info-container",
+      "format": "markdown",
+      "required": false
+    }
+  ]
+}