internal-web-reader 0.1.0__tar.gz → 0.2.0__tar.gz

{internal_web_reader-0.1.0 → internal_web_reader-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: internal-web-reader
-Version: 0.1.0
+Version: 0.2.0
 Summary: MCP Server - let Claude Code read internal web pages via browser cookies
 Author: zhanghaoran18006
 License-Expression: MIT
@@ -12,10 +12,9 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: mcp[cli]<2.0.0,>=1.2.0
-Requires-Dist: httpx>=0.27.0
+Requires-Dist: playwright>=1.40.0
 Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: html2text>=2024.2.26
-Requires-Dist: browser_cookie3>=0.19.1
 
 # Internal Web Reader
 

{internal_web_reader-0.1.0 → internal_web_reader-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "internal-web-reader"
-version = "0.1.0"
+version = "0.2.0"
 description = "MCP Server - let Claude Code read internal web pages via browser cookies"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -19,10 +19,9 @@ classifiers = [
 ]
 dependencies = [
     "mcp[cli]>=1.2.0,<2.0.0",
-    "httpx>=0.27.0",
+    "playwright>=1.40.0",
     "beautifulsoup4>=4.12.0",
     "html2text>=2024.2.26",
-    "browser_cookie3>=0.19.1",
 ]
 
 [project.scripts]

internal_web_reader-0.2.0/src/internal_web_reader/__main__.py

@@ -0,0 +1,55 @@
+"""MCP Server 入口。用 Playwright 浏览器读网页，登录一次自动保存。"""
+
+import atexit
+import logging
+
+from mcp.server.fastmcp import FastMCP
+
+from .reader import read_page, close_browser
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[web-reader] %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+log = logging.getLogger("internal_web_reader")
+
+atexit.register(close_browser)
+
+mcp = FastMCP("internal-web-reader")
+
+
+@mcp.tool()
+def read_page_tool(url: str) -> str:
+    """读取任意网页，返回干净的 Markdown。
+    首次访问需要登录的网站时，会弹出浏览器窗口让你登录。
+    登录一次后 session 自动保存，后续无需重复登录。
+    """
+    page = read_page(url)
+
+    lines = [f"# {page.title}", f"> {page.url}", "", page.content]
+    if 0 < len(page.links) <= 30:
+        lines += ["", "---", "## 页面链接"]
+        for lk in page.links:
+            lines.append(f"- [{lk['text']}]({lk['href']})")
+    return "\n".join(lines)
+
+
+@mcp.tool()
+def list_links(url: str) -> str:
+    """列出页面上的所有链接，用于浏览内网站点。"""
+    page = read_page(url)
+    if not page.links:
+        return f"页面 {url} 上未找到链接。"
+    lines = [f"# {page.title} 的链接", f"> {len(page.links)} 个", ""]
+    for lk in page.links:
+        lines.append(f"- [{lk['text']}]({lk['href']})")
+    return "\n".join(lines)
+
+
+def main():
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

internal_web_reader-0.2.0/src/internal_web_reader/reader.py

@@ -0,0 +1,166 @@
+"""用 Playwright 启动浏览器读网页。登录一次，session 自动保存。"""
+
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+import html2text
+
+log = logging.getLogger("internal_web_reader")
+
+_H2T = html2text.HTML2Text()
+_H2T.body_width = 0
+_H2T.ignore_links = False
+_H2T.ignore_images = False
+_H2T.protect_links = True
+_H2T.wrap_links = False
+_H2T.unicode_snob = True
+
+# 持久化浏览器 profile 目录
+_USER_DATA_DIR = os.path.join(
+    os.environ.get("APPDATA", os.path.expanduser("~")),
+    "internal-web-reader",
+    "browser-profile",
+)
+
+
+@dataclass
+class PageResult:
+    url: str
+    title: str
+    content: str
+    links: list[dict] = field(default_factory=list)
+
+
+# 全局浏览器实例（进程内复用）
+_browser = None
+_context = None
+_pw = None
+
+
+def _ensure_browser():
+    """启动 Playwright 浏览器（进程内单例）。"""
+    global _browser, _context, _pw
+    if _context is not None:
+        return _context
+
+    from playwright.sync_api import sync_playwright
+
+    _pw = sync_playwright().start()
+    os.makedirs(_USER_DATA_DIR, exist_ok=True)
+
+    _context = _pw.chromium.launch_persistent_context(
+        _USER_DATA_DIR,
+        headless=False,
+        channel="chrome",
+        viewport={"width": 1280, "height": 900},
+        locale="zh-CN",
+        args=["--disable-blink-features=AutomationControlled"],
+    )
+    log.info("浏览器已启动")
+    return _context
+
+
+def _is_login_page(page) -> bool:
+    """判断当前页面是否是登录页。"""
+    url = page.url.lower()
+    if any(k in url for k in ("/login", "/signin", "/sso", "/auth", "/cas/", "/oauth")):
+        return True
+    # 检查页面是否有密码输入框
+    pw_inputs = page.query_selector_all('input[type="password"]')
+    if pw_inputs and len(pw_inputs) > 0:
+        return True
+    return False
+
+
+def read_page(url: str, timeout: float = 60) -> PageResult:
+    """用浏览器打开 URL，返回清洗后的 Markdown。"""
+    ctx = _ensure_browser()
+
+    # 用现有 page 或新建
+    page = ctx.pages[0] if ctx.pages else ctx.new_page()
+
+    page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+
+    # 如果是登录页，等用户手动登录
+    if _is_login_page(page):
+        log.info("检测到登录页面，请在浏览器窗口中完成登录...")
+        # 等 URL 变化（登录成功后会跳转）或者最多等 120 秒
+        try:
+            page.wait_for_url(
+                lambda u: u != page.url and "login" not in u.lower(),
+                timeout=120_000,
+            )
+            # 等页面加载完
+            page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            log.info(f"登录成功，当前页面: {page.url}")
+        except Exception:
+            log.warning("等待登录超时，尝试继续...")
+
+    # 获取页面内容
+    final_url = page.url
+    html = page.content()
+    title = page.title() or url
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # 去噪
+    for tag in soup.find_all(["script", "style", "noscript", "iframe", "svg", "nav", "footer", "aside"]):
+        tag.decompose()
+
+    # 找主体
+    main = None
+    for sel in ("article", "main", '[role="main"]', ".content", ".article-content", ".markdown-body", "#content", "#main"):
+        el = soup.select_one(sel)
+        if el and len(el.get_text(strip=True)) > 100:
+            main = el
+            break
+    target = main or soup.body or soup
+
+    links = _extract_links(target, final_url)
+    markdown = _H2T.handle(str(target)).strip()
+
+    if len(markdown) > 200_000:
+        markdown = markdown[:200_000] + "\n\n... [已截断]"
+
+    return PageResult(url=final_url, title=title, content=markdown, links=links)
+
+
+def close_browser():
+    """关闭浏览器（进程退出时调用）。"""
+    global _browser, _context, _pw
+    if _context:
+        try:
+            _context.close()
+        except Exception:
+            pass
+    if _pw:
+        try:
+            _pw.stop()
+        except Exception:
+            pass
+    _context = None
+    _pw = None
+
+
+def _extract_links(soup, base_url: str) -> list[dict]:
+    seen: set[str] = set()
+    links: list[dict] = []
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
+            continue
+        try:
+            absolute = urljoin(base_url, href)
+        except Exception:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        text = a.get_text(strip=True)[:100] or href
+        links.append({"text": text, "href": absolute})
+    return links

{internal_web_reader-0.1.0 → internal_web_reader-0.2.0}/src/internal_web_reader.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: internal-web-reader
-Version: 0.1.0
+Version: 0.2.0
 Summary: MCP Server - let Claude Code read internal web pages via browser cookies
 Author: zhanghaoran18006
 License-Expression: MIT
@@ -12,10 +12,9 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: mcp[cli]<2.0.0,>=1.2.0
-Requires-Dist: httpx>=0.27.0
+Requires-Dist: playwright>=1.40.0
 Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: html2text>=2024.2.26
-Requires-Dist: browser_cookie3>=0.19.1
 
 # Internal Web Reader
 

{internal_web_reader-0.1.0 → internal_web_reader-0.2.0}/src/internal_web_reader.egg-info/SOURCES.txt

@@ -2,7 +2,6 @@ README.md
 pyproject.toml
 src/internal_web_reader/__init__.py
 src/internal_web_reader/__main__.py
-src/internal_web_reader/cookies.py
 src/internal_web_reader/reader.py
 src/internal_web_reader.egg-info/PKG-INFO
 src/internal_web_reader.egg-info/SOURCES.txt

{internal_web_reader-0.1.0 → internal_web_reader-0.2.0}/src/internal_web_reader.egg-info/requires.txt

@@ -1,5 +1,4 @@
 mcp[cli]<2.0.0,>=1.2.0
-httpx>=0.27.0
+playwright>=1.40.0
 beautifulsoup4>=4.12.0
 html2text>=2024.2.26
-browser_cookie3>=0.19.1

internal_web_reader-0.1.0/src/internal_web_reader/__main__.py

@@ -1,129 +0,0 @@
-"""MCP Server 入口。Cookie 懒加载：首次访问某域名时自动从浏览器提取。"""
-
-import logging
-from urllib.parse import urlparse
-
-from mcp.server.fastmcp import FastMCP
-
-from .cookies import get_cookie_header
-from .reader import read_page
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[web-reader] %(message)s",
-    handlers=[logging.StreamHandler()],
-)
-log = logging.getLogger("internal_web_reader")
-
-# domain → cookie header string（内存缓存）
-_cookie_cache: dict[str, str] = {}
-
-mcp = FastMCP("internal-web-reader")
-
-
-def _get_cookie(url: str) -> str | None:
-    """获取 URL 对应的 Cookie，没有就自动从浏览器提取并缓存。"""
-    try:
-        host = urlparse(url).hostname or ""
-    except Exception:
-        return None
-
-    if not host:
-        return None
-
-    # 1. 缓存命中
-    if host in _cookie_cache:
-        return _cookie_cache[host]
-
-    # 2. 父域名
-    parts = host.split(".")
-    for i in range(1, len(parts) - 1):
-        parent = ".".join(parts[i:])
-        if parent in _cookie_cache:
-            return _cookie_cache[parent]
-
-    # 3. 自动提取
-    header = get_cookie_header(host)
-    if header:
-        _cookie_cache[host] = header
-    return header
-
-
-def _refresh_cookie(host: str) -> str | None:
-    """强制刷新某域名的 Cookie（401/403 时调用）。"""
-    _cookie_cache.pop(host, None)
-    parts = host.split(".")
-    for i in range(1, len(parts) - 1):
-        _cookie_cache.pop(".".join(parts[i:]), None)
-
-    header = get_cookie_header(host)
-    if header:
-        _cookie_cache[host] = header
-    return header
-
-
-# ── Tools ────────────────────────────────────────────────
-
-
-@mcp.tool()
-def read_page_tool(url: str) -> str:
-    """读取任意网页，返回干净的 Markdown。
-    Cookie 自动从浏览器提取，不需要预配置。
-    支持内网 GitLab、Wiki、文档中心等。
-    """
-    cookie = _get_cookie(url)
-
-    try:
-        page = read_page(url, cookie=cookie)
-    except Exception as e:
-        # 401/403 → 刷新 Cookie 重试
-        msg = str(e)
-        if any(k in msg for k in ("401", "403", "Unauthorized", "Forbidden")):
-            try:
-                host = urlparse(url).hostname or ""
-            except Exception:
-                raise
-            log.info(f"鉴权失败，刷新 {host} Cookie...")
-            cookie = _refresh_cookie(host)
-            if cookie:
-                log.info("重试中...")
-                page = read_page(url, cookie=cookie)
-            else:
-                raise
-        else:
-            raise
-
-    lines = [f"# {page.title}", f"> {page.url}", "", page.content]
-
-    if 0 < len(page.links) <= 30:
-        lines += ["", "---", "## 页面链接"]
-        for lk in page.links:
-            lines.append(f"- [{lk['text']}]({lk['href']})")
-
-    return "\n".join(lines)
-
-
-@mcp.tool()
-def list_links(url: str) -> str:
-    """列出页面上的所有链接，用于浏览内网站点。"""
-    cookie = _get_cookie(url)
-    page = read_page(url, cookie=cookie)
-
-    if not page.links:
-        return f"页面 {url} 上未找到链接。"
-
-    lines = [f"# {page.title} 的链接", f"> {len(page.links)} 个", ""]
-    for lk in page.links:
-        lines.append(f"- [{lk['text']}]({lk['href']})")
-    return "\n".join(lines)
-
-
-# ── Entry ────────────────────────────────────────────────
-
-
-def main():
-    mcp.run()
-
-
-if __name__ == "__main__":
-    main()

internal_web_reader-0.1.0/src/internal_web_reader/cookies.py

@@ -1,44 +0,0 @@
-"""从本地浏览器自动提取 Cookie。"""
-
-import logging
-
-log = logging.getLogger("internal_web_reader")
-
-
-def get_cookie_header(domain: str) -> str | None:
-    """
-    从 Chrome / Edge / Firefox 提取指定域名的 Cookie，
-    返回 "name1=val1; name2=val2" 格式的字符串。
-    找不到返回 None。
-    """
-    cookie_jars = []
-
-    # 依次尝试各浏览器
-    try:
-        import browser_cookie3
-
-        for loader in (browser_cookie3.chrome, browser_cookie3.edge, browser_cookie3.firefox):
-            try:
-                cj = loader(domain_name=domain)
-                cookie_jars.append(cj)
-            except Exception as e:
-                log.debug(f"{loader.__name__} 跳过: {e}")
-    except ImportError:
-        log.warning("browser_cookie3 未安装，无法自动提取 Cookie")
-        return None
-
-    # 合并 + 去重
-    seen: dict[str, str] = {}
-    for cj in cookie_jars:
-        for c in cj:
-            # 匹配: 精确域名 或 父域名 (.example.com 匹配 sub.example.com)
-            cdomain = c.domain.lstrip(".")
-            if domain == cdomain or domain.endswith("." + cdomain):
-                seen[c.name] = c.value
-
-    if not seen:
-        return None
-
-    header = "; ".join(f"{k}={v}" for k, v in seen.items())
-    log.info(f"从浏览器提取 {len(seen)} 个 Cookie ({domain})")
-    return header

internal_web_reader-0.1.0/src/internal_web_reader/reader.py

@@ -1,114 +0,0 @@
-"""抓取网页，HTML → 干净 Markdown。"""
-
-import logging
-from dataclasses import dataclass, field
-from urllib.parse import urljoin
-
-import httpx
-from bs4 import BeautifulSoup
-import html2text
-
-log = logging.getLogger("internal_web_reader")
-
-_H2T = html2text.HTML2Text()
-_H2T.body_width = 0  # 不自动换行
-_H2T.ignore_links = False
-_H2T.ignore_images = False
-_H2T.protect_links = True
-_H2T.wrap_links = False
-_H2T.unicode_snob = True
-
-_UA = (
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/124.0.0.0 Safari/537.36"
-)
-
-
-@dataclass
-class PageResult:
-    url: str
-    title: str
-    content: str
-    links: list[dict] = field(default_factory=list)
-
-
-def read_page(url: str, cookie: str | None = None, timeout: float = 30) -> PageResult:
-    """抓取一个 URL，返回清洗后的 Markdown。"""
-
-    headers = {
-        "User-Agent": _UA,
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-    }
-    if cookie:
-        headers["Cookie"] = cookie
-
-    resp = httpx.get(url, headers=headers, follow_redirects=True, timeout=timeout)
-    resp.raise_for_status()
-
-    ct = resp.headers.get("content-type", "")
-    if "html" not in ct and "text" not in ct:
-        return PageResult(url=url, title=url, content=f"[非 HTML] Content-Type: {ct}")
-
-    # 检测编码
-    resp.encoding = resp.charset_encoding or "utf-8"
-    html = resp.text
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    # 标题
-    title = ""
-    if soup.title and soup.title.string:
-        title = soup.title.string.strip()
-    if not title:
-        h1 = soup.find("h1")
-        if h1:
-            title = h1.get_text(strip=True)
-    title = title or url
-
-    # 去噪
-    for tag in soup.find_all(["script", "style", "noscript", "iframe", "svg", "nav", "footer", "aside"]):
-        tag.decompose()
-
-    # 找主体内容
-    main = None
-    for sel in ("article", "main", '[role="main"]', ".content", ".article-content", ".markdown-body", "#content", "#main"):
-        el = soup.select_one(sel)
-        if el and len(el.get_text(strip=True)) > 100:
-            main = el
-            break
-
-    target = main or soup.body or soup
-
-    # 提取链接
-    links = _extract_links(target, url)
-
-    # HTML → Markdown
-    content_html = str(target)
-    markdown = _H2T.handle(content_html).strip()
-
-    # 截断
-    if len(markdown) > 200_000:
-        markdown = markdown[:200_000] + "\n\n... [内容过长，已截断]"
-
-    return PageResult(url=url, title=title, content=markdown, links=links)
-
-
-def _extract_links(soup, base_url: str) -> list[dict]:
-    seen: set[str] = set()
-    links: list[dict] = []
-    for a in soup.find_all("a", href=True):
-        href = a["href"].strip()
-        if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
-            continue
-        try:
-            absolute = urljoin(base_url, href)
-        except Exception:
-            continue
-        if absolute in seen:
-            continue
-        seen.add(absolute)
-        text = a.get_text(strip=True)[:100] or href
-        links.append({"text": text, "href": absolute})
-    return links