internal-web-reader 0.1.0__py3-none-any.whl

internal_web_reader/__init__.py

@@ -0,0 +1,3 @@
+"""Internal Web Reader — MCP Server for Claude Code."""
+
+__version__ = "0.1.0"

internal_web_reader/__main__.py

@@ -0,0 +1,129 @@
+"""MCP Server 入口。Cookie 懒加载：首次访问某域名时自动从浏览器提取。"""
+
+import logging
+from urllib.parse import urlparse
+
+from mcp.server.fastmcp import FastMCP
+
+from .cookies import get_cookie_header
+from .reader import read_page
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[web-reader] %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+log = logging.getLogger("internal_web_reader")
+
+# domain → cookie header string（内存缓存）
+_cookie_cache: dict[str, str] = {}
+
+mcp = FastMCP("internal-web-reader")
+
+
+def _get_cookie(url: str) -> str | None:
+    """获取 URL 对应的 Cookie，没有就自动从浏览器提取并缓存。"""
+    try:
+        host = urlparse(url).hostname or ""
+    except Exception:
+        return None
+
+    if not host:
+        return None
+
+    # 1. 缓存命中
+    if host in _cookie_cache:
+        return _cookie_cache[host]
+
+    # 2. 父域名
+    parts = host.split(".")
+    for i in range(1, len(parts) - 1):
+        parent = ".".join(parts[i:])
+        if parent in _cookie_cache:
+            return _cookie_cache[parent]
+
+    # 3. 自动提取
+    header = get_cookie_header(host)
+    if header:
+        _cookie_cache[host] = header
+    return header
+
+
+def _refresh_cookie(host: str) -> str | None:
+    """强制刷新某域名的 Cookie（401/403 时调用）。"""
+    _cookie_cache.pop(host, None)
+    parts = host.split(".")
+    for i in range(1, len(parts) - 1):
+        _cookie_cache.pop(".".join(parts[i:]), None)
+
+    header = get_cookie_header(host)
+    if header:
+        _cookie_cache[host] = header
+    return header
+
+
+# ── Tools ────────────────────────────────────────────────
+
+
+@mcp.tool()
+def read_page_tool(url: str) -> str:
+    """读取任意网页，返回干净的 Markdown。
+    Cookie 自动从浏览器提取，不需要预配置。
+    支持内网 GitLab、Wiki、文档中心等。
+    """
+    cookie = _get_cookie(url)
+
+    try:
+        page = read_page(url, cookie=cookie)
+    except Exception as e:
+        # 401/403 → 刷新 Cookie 重试
+        msg = str(e)
+        if any(k in msg for k in ("401", "403", "Unauthorized", "Forbidden")):
+            try:
+                host = urlparse(url).hostname or ""
+            except Exception:
+                raise
+            log.info(f"鉴权失败，刷新 {host} Cookie...")
+            cookie = _refresh_cookie(host)
+            if cookie:
+                log.info("重试中...")
+                page = read_page(url, cookie=cookie)
+            else:
+                raise
+        else:
+            raise
+
+    lines = [f"# {page.title}", f"> {page.url}", "", page.content]
+
+    if 0 < len(page.links) <= 30:
+        lines += ["", "---", "## 页面链接"]
+        for lk in page.links:
+            lines.append(f"- [{lk['text']}]({lk['href']})")
+
+    return "\n".join(lines)
+
+
+@mcp.tool()
+def list_links(url: str) -> str:
+    """列出页面上的所有链接，用于浏览内网站点。"""
+    cookie = _get_cookie(url)
+    page = read_page(url, cookie=cookie)
+
+    if not page.links:
+        return f"页面 {url} 上未找到链接。"
+
+    lines = [f"# {page.title} 的链接", f"> {len(page.links)} 个", ""]
+    for lk in page.links:
+        lines.append(f"- [{lk['text']}]({lk['href']})")
+    return "\n".join(lines)
+
+
+# ── Entry ────────────────────────────────────────────────
+
+
+def main():
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

internal_web_reader/cookies.py

@@ -0,0 +1,44 @@
+"""从本地浏览器自动提取 Cookie。"""
+
+import logging
+
+log = logging.getLogger("internal_web_reader")
+
+
+def get_cookie_header(domain: str) -> str | None:
+    """
+    从 Chrome / Edge / Firefox 提取指定域名的 Cookie，
+    返回 "name1=val1; name2=val2" 格式的字符串。
+    找不到返回 None。
+    """
+    cookie_jars = []
+
+    # 依次尝试各浏览器
+    try:
+        import browser_cookie3
+
+        for loader in (browser_cookie3.chrome, browser_cookie3.edge, browser_cookie3.firefox):
+            try:
+                cj = loader(domain_name=domain)
+                cookie_jars.append(cj)
+            except Exception as e:
+                log.debug(f"{loader.__name__} 跳过: {e}")
+    except ImportError:
+        log.warning("browser_cookie3 未安装，无法自动提取 Cookie")
+        return None
+
+    # 合并 + 去重
+    seen: dict[str, str] = {}
+    for cj in cookie_jars:
+        for c in cj:
+            # 匹配: 精确域名 或 父域名 (.example.com 匹配 sub.example.com)
+            cdomain = c.domain.lstrip(".")
+            if domain == cdomain or domain.endswith("." + cdomain):
+                seen[c.name] = c.value
+
+    if not seen:
+        return None
+
+    header = "; ".join(f"{k}={v}" for k, v in seen.items())
+    log.info(f"从浏览器提取 {len(seen)} 个 Cookie ({domain})")
+    return header

internal_web_reader/reader.py

@@ -0,0 +1,114 @@
+"""抓取网页，HTML → 干净 Markdown。"""
+
+import logging
+from dataclasses import dataclass, field
+from urllib.parse import urljoin
+
+import httpx
+from bs4 import BeautifulSoup
+import html2text
+
+log = logging.getLogger("internal_web_reader")
+
+_H2T = html2text.HTML2Text()
+_H2T.body_width = 0  # 不自动换行
+_H2T.ignore_links = False
+_H2T.ignore_images = False
+_H2T.protect_links = True
+_H2T.wrap_links = False
+_H2T.unicode_snob = True
+
+_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+
+
+@dataclass
+class PageResult:
+    url: str
+    title: str
+    content: str
+    links: list[dict] = field(default_factory=list)
+
+
+def read_page(url: str, cookie: str | None = None, timeout: float = 30) -> PageResult:
+    """抓取一个 URL，返回清洗后的 Markdown。"""
+
+    headers = {
+        "User-Agent": _UA,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    }
+    if cookie:
+        headers["Cookie"] = cookie
+
+    resp = httpx.get(url, headers=headers, follow_redirects=True, timeout=timeout)
+    resp.raise_for_status()
+
+    ct = resp.headers.get("content-type", "")
+    if "html" not in ct and "text" not in ct:
+        return PageResult(url=url, title=url, content=f"[非 HTML] Content-Type: {ct}")
+
+    # 检测编码
+    resp.encoding = resp.charset_encoding or "utf-8"
+    html = resp.text
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # 标题
+    title = ""
+    if soup.title and soup.title.string:
+        title = soup.title.string.strip()
+    if not title:
+        h1 = soup.find("h1")
+        if h1:
+            title = h1.get_text(strip=True)
+    title = title or url
+
+    # 去噪
+    for tag in soup.find_all(["script", "style", "noscript", "iframe", "svg", "nav", "footer", "aside"]):
+        tag.decompose()
+
+    # 找主体内容
+    main = None
+    for sel in ("article", "main", '[role="main"]', ".content", ".article-content", ".markdown-body", "#content", "#main"):
+        el = soup.select_one(sel)
+        if el and len(el.get_text(strip=True)) > 100:
+            main = el
+            break
+
+    target = main or soup.body or soup
+
+    # 提取链接
+    links = _extract_links(target, url)
+
+    # HTML → Markdown
+    content_html = str(target)
+    markdown = _H2T.handle(content_html).strip()
+
+    # 截断
+    if len(markdown) > 200_000:
+        markdown = markdown[:200_000] + "\n\n... [内容过长，已截断]"
+
+    return PageResult(url=url, title=title, content=markdown, links=links)
+
+
+def _extract_links(soup, base_url: str) -> list[dict]:
+    seen: set[str] = set()
+    links: list[dict] = []
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
+            continue
+        try:
+            absolute = urljoin(base_url, href)
+        except Exception:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        text = a.get_text(strip=True)[:100] or href
+        links.append({"text": text, "href": absolute})
+    return links

internal_web_reader-0.1.0.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: internal-web-reader
+Version: 0.1.0
+Summary: MCP Server - let Claude Code read internal web pages via browser cookies
+Author: zhanghaoran18006
+License-Expression: MIT
+Keywords: mcp,claude,internal,wiki,confluence,gitlab
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: mcp[cli]<2.0.0,>=1.2.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: html2text>=2024.2.26
+Requires-Dist: browser_cookie3>=0.19.1
+
+# Internal Web Reader
+
+让 Claude Code 能读内网网页。
+
+## 安装
+
+```
+pip install internal-web-reader
+```
+
+## 配置 Claude Code
+
+```
+claude mcp add -s user internal-web-reader -- internal-web-reader
+```
+
+## 使用
+
+在 Claude Code 里直接给链接：
+
+```
+> 看看 https://git.corpautohome.com/xxx/-/issues/123
+> 读一下 https://doc.autohome.com.cn/docapi/page/share/share_1Hx8joJJIfo
+```
+
+Cookie 自动从浏览器提取。前提：浏览器里登录过、VPN 连着。

internal_web_reader-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+internal_web_reader/__init__.py,sha256=EUOrBX7nMn6Rt-PgPHWBEjuORtXuWfQuuey2hNFzFT8,81
+internal_web_reader/__main__.py,sha256=YxSIfjaCroshuvjqaSqcZG-7iRzYnVOn_gwLBuMOjak,3628
+internal_web_reader/cookies.py,sha256=3HEA1fSjor_BqrnntRpQlLKUERCuBewy9sJRoPl7pFI,1363
+internal_web_reader/reader.py,sha256=t1hmIMRONSyBHftghpqkCbM-PPt0Z6uGNdGJgxQzG70,3246
+internal_web_reader-0.1.0.dist-info/METADATA,sha256=hifEi16FrM3aTJ3m8zzUGDy2J5n-W3yYkc8NGGF-_zs,1211
+internal_web_reader-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+internal_web_reader-0.1.0.dist-info/entry_points.txt,sha256=iEhJm-03JedWF3i1Awbegyg1u2-YEF_TYC0QGORzV24,74
+internal_web_reader-0.1.0.dist-info/top_level.txt,sha256=jhE9xwuy3_-TQU3BWs3xkH32SA_1bYFDEA1ArtYGnZ8,20
+internal_web_reader-0.1.0.dist-info/RECORD,,

internal_web_reader-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

internal_web_reader-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+internal-web-reader = internal_web_reader.__main__:main

internal_web_reader-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+internal_web_reader