internal-web-reader 0.2.1__tar.gz → 0.2.3__tar.gz

{internal_web_reader-0.2.1 → internal_web_reader-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: internal-web-reader
-Version: 0.2.1
+Version: 0.2.3
 Summary: MCP Server - let Claude Code read internal web pages via browser cookies
 Author: zhanghaoran18006
 License-Expression: MIT

{internal_web_reader-0.2.1 → internal_web_reader-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "internal-web-reader"
-version = "0.2.1"
+version = "0.2.3"
 description = "MCP Server - let Claude Code read internal web pages via browser cookies"
 readme = "README.md"
 requires-python = ">=3.10"

internal_web_reader-0.2.3/src/internal_web_reader/__main__.py

@@ -0,0 +1,54 @@
+"""MCP Server 入口。"""
+
+import atexit
+import logging
+
+from mcp.server.fastmcp import FastMCP
+
+from .reader import read_page, close_browser
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[web-reader] %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+mcp = FastMCP("internal-web-reader")
+
+
+@mcp.tool()
+async def read_web_page(url: str) -> str:
+    """[READ WEB PAGES] Open any URL in a local Chrome browser and return Markdown content.
+    USE THIS TOOL for all web page reading, especially internal/intranet sites
+    (GitLab, Wiki, docs requiring VPN/SSO). Do NOT use the built-in Fetch tool for web pages.
+    First visit to a login-required site will pop up a browser window for you to log in.
+    Session is saved automatically for future visits.
+    """
+    page = await read_page(url)
+
+    lines = [f"# {page.title}", f"> {page.url}", "", page.content]
+    if 0 < len(page.links) <= 30:
+        lines += ["", "---", "## Links"]
+        for lk in page.links:
+            lines.append(f"- [{lk['text']}]({lk['href']})")
+    return "\n".join(lines)
+
+
+@mcp.tool()
+async def list_web_links(url: str) -> str:
+    """List all links on a web page for navigation."""
+    page = await read_page(url)
+    if not page.links:
+        return f"No links found on {url}."
+    lines = [f"# {page.title} - Links", f"> {len(page.links)} links", ""]
+    for lk in page.links:
+        lines.append(f"- [{lk['text']}]({lk['href']})")
+    return "\n".join(lines)
+
+
+def main():
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

internal_web_reader-0.2.3/src/internal_web_reader/reader.py

@@ -0,0 +1,182 @@
+"""用 Playwright (async) 启动浏览器读网页。登录一次，session 自动保存。"""
+
+import asyncio
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+import html2text
+
+log = logging.getLogger("internal_web_reader")
+
+_H2T = html2text.HTML2Text()
+_H2T.body_width = 0
+_H2T.ignore_links = False
+_H2T.ignore_images = False
+_H2T.protect_links = True
+_H2T.wrap_links = False
+_H2T.unicode_snob = True
+
+_USER_DATA_DIR = os.path.join(
+    os.environ.get("APPDATA", os.path.expanduser("~")),
+    "internal-web-reader",
+    "browser-profile",
+)
+
+
+@dataclass
+class PageResult:
+    url: str
+    title: str
+    content: str
+    links: list[dict] = field(default_factory=list)
+
+
+_context = None
+_pw = None
+
+
+def _stderr(msg: str):
+    sys.stderr.write(f"\n{'='*60}\n{msg}\n{'='*60}\n")
+    sys.stderr.flush()
+
+
+async def _ensure_browser():
+    global _context, _pw
+    if _context is not None:
+        return _context
+
+    from playwright.async_api import async_playwright
+
+    _pw = await async_playwright().start()
+    os.makedirs(_USER_DATA_DIR, exist_ok=True)
+
+    _context = await _pw.chromium.launch_persistent_context(
+        _USER_DATA_DIR,
+        headless=False,
+        channel="chrome",
+        viewport={"width": 1280, "height": 900},
+        locale="zh-CN",
+        args=["--disable-blink-features=AutomationControlled"],
+    )
+    return _context
+
+
+def _is_login_url(url: str) -> bool:
+    url_lower = url.lower()
+    keywords = ["/login", "/signin", "/sso", "/cas", "/oauth", "/auth", "passport"]
+    return any(k in url_lower for k in keywords)
+
+
+async def _wait_for_login(page, timeout_s: int = 300):
+    _stderr(
+        "  需要登录！请切换到弹出的浏览器窗口完成登录。\n"
+        "  登录成功后会自动继续，不用回来操作终端。"
+    )
+
+    await page.evaluate("""() => {
+        const b = document.createElement('div');
+        b.id = '__iwr__';
+        b.style.cssText = 'position:fixed;top:0;left:0;right:0;z-index:999999;'
+            + 'background:#ff6b35;color:white;padding:16px;text-align:center;'
+            + 'font-size:18px;font-weight:bold;font-family:sans-serif;'
+            + 'box-shadow:0 4px 12px rgba(0,0,0,0.3);';
+        b.textContent = 'Internal Web Reader: Please log in here. Auto-continuing after login...';
+        document.body.prepend(b);
+    }""")
+
+    elapsed = 0
+    while elapsed < timeout_s:
+        await asyncio.sleep(2)
+        elapsed += 2
+
+        current_url = page.url.lower()
+        has_pw = await page.query_selector('input[type="password"]')
+
+        if not _is_login_url(current_url) and not has_pw:
+            await page.evaluate("() => { const e = document.getElementById('__iwr__'); if(e) e.remove(); }")
+            _stderr("  登录成功！正在读取页面...")
+            await page.wait_for_load_state("domcontentloaded", timeout=30_000)
+            return True
+
+    _stderr("  登录等待超时，尝试继续...")
+    return False
+
+
+async def read_page(url: str, timeout: float = 60) -> PageResult:
+    ctx = await _ensure_browser()
+    page = ctx.pages[0] if ctx.pages else await ctx.new_page()
+
+    try:
+        await page.bring_to_front()
+    except Exception:
+        pass
+
+    await page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+
+    # 登录检测
+    if _is_login_url(page.url) or await page.query_selector('input[type="password"]'):
+        await _wait_for_login(page)
+
+    final_url = page.url
+    html = await page.content()
+    title = await page.title() or url
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    for tag in soup.find_all(["script", "style", "noscript", "iframe", "svg", "nav", "footer", "aside"]):
+        tag.decompose()
+
+    main = None
+    for sel in ("article", "main", '[role="main"]', ".content", ".article-content", ".markdown-body", "#content", "#main"):
+        el = soup.select_one(sel)
+        if el and len(el.get_text(strip=True)) > 100:
+            main = el
+            break
+    target = main or soup.body or soup
+
+    links = _extract_links(target, final_url)
+    markdown = _H2T.handle(str(target)).strip()
+
+    if len(markdown) > 200_000:
+        markdown = markdown[:200_000] + "\n\n... [truncated]"
+
+    return PageResult(url=final_url, title=title, content=markdown, links=links)
+
+
+async def close_browser():
+    global _context, _pw
+    if _context:
+        try:
+            await _context.close()
+        except Exception:
+            pass
+    if _pw:
+        try:
+            await _pw.stop()
+        except Exception:
+            pass
+    _context = None
+    _pw = None
+
+
+def _extract_links(soup, base_url: str) -> list[dict]:
+    seen: set[str] = set()
+    links: list[dict] = []
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
+            continue
+        try:
+            absolute = urljoin(base_url, href)
+        except Exception:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        text = a.get_text(strip=True)[:100] or href
+        links.append({"text": text, "href": absolute})
+    return links

{internal_web_reader-0.2.1 → internal_web_reader-0.2.3}/src/internal_web_reader.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: internal-web-reader
-Version: 0.2.1
+Version: 0.2.3
 Summary: MCP Server - let Claude Code read internal web pages via browser cookies
 Author: zhanghaoran18006
 License-Expression: MIT

internal_web_reader-0.2.1/src/internal_web_reader/__main__.py

@@ -1,56 +0,0 @@
-"""MCP Server 入口。用 Playwright 浏览器读网页，登录一次自动保存。"""
-
-import atexit
-import logging
-
-from mcp.server.fastmcp import FastMCP
-
-from .reader import read_page, close_browser
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[web-reader] %(message)s",
-    handlers=[logging.StreamHandler()],
-)
-log = logging.getLogger("internal_web_reader")
-
-atexit.register(close_browser)
-
-mcp = FastMCP("internal-web-reader")
-
-
-@mcp.tool()
-def read_web_page(url: str) -> str:
-    """【读网页用这个】通过本地浏览器打开任意 URL 并返回 Markdown 内容。
-    专门用于访问公司内网页面（GitLab、Wiki、文档中心等需要 VPN 的网站）。
-    也适用于任何公开网页。首次访问需登录的站点会弹出浏览器窗口完成登录，
-    之后 session 自动保存。优先使用此工具而非内置 Fetch 来读取网页。
-    """
-    page = read_page(url)
-
-    lines = [f"# {page.title}", f"> {page.url}", "", page.content]
-    if 0 < len(page.links) <= 30:
-        lines += ["", "---", "## 页面链接"]
-        for lk in page.links:
-            lines.append(f"- [{lk['text']}]({lk['href']})")
-    return "\n".join(lines)
-
-
-@mcp.tool()
-def list_web_links(url: str) -> str:
-    """列出网页上的所有链接，用于浏览内网站点结构。"""
-    page = read_page(url)
-    if not page.links:
-        return f"页面 {url} 上未找到链接。"
-    lines = [f"# {page.title} 的链接", f"> {len(page.links)} 个", ""]
-    for lk in page.links:
-        lines.append(f"- [{lk['text']}]({lk['href']})")
-    return "\n".join(lines)
-
-
-def main():
-    mcp.run()
-
-
-if __name__ == "__main__":
-    main()

internal_web_reader-0.2.1/src/internal_web_reader/reader.py

@@ -1,166 +0,0 @@
-"""用 Playwright 启动浏览器读网页。登录一次，session 自动保存。"""
-
-import logging
-import os
-import re
-from dataclasses import dataclass, field
-from pathlib import Path
-from urllib.parse import urljoin
-
-from bs4 import BeautifulSoup
-import html2text
-
-log = logging.getLogger("internal_web_reader")
-
-_H2T = html2text.HTML2Text()
-_H2T.body_width = 0
-_H2T.ignore_links = False
-_H2T.ignore_images = False
-_H2T.protect_links = True
-_H2T.wrap_links = False
-_H2T.unicode_snob = True
-
-# 持久化浏览器 profile 目录
-_USER_DATA_DIR = os.path.join(
-    os.environ.get("APPDATA", os.path.expanduser("~")),
-    "internal-web-reader",
-    "browser-profile",
-)
-
-
-@dataclass
-class PageResult:
-    url: str
-    title: str
-    content: str
-    links: list[dict] = field(default_factory=list)
-
-
-# 全局浏览器实例（进程内复用）
-_browser = None
-_context = None
-_pw = None
-
-
-def _ensure_browser():
-    """启动 Playwright 浏览器（进程内单例）。"""
-    global _browser, _context, _pw
-    if _context is not None:
-        return _context
-
-    from playwright.sync_api import sync_playwright
-
-    _pw = sync_playwright().start()
-    os.makedirs(_USER_DATA_DIR, exist_ok=True)
-
-    _context = _pw.chromium.launch_persistent_context(
-        _USER_DATA_DIR,
-        headless=False,
-        channel="chrome",
-        viewport={"width": 1280, "height": 900},
-        locale="zh-CN",
-        args=["--disable-blink-features=AutomationControlled"],
-    )
-    log.info("浏览器已启动")
-    return _context
-
-
-def _is_login_page(page) -> bool:
-    """判断当前页面是否是登录页。"""
-    url = page.url.lower()
-    if any(k in url for k in ("/login", "/signin", "/sso", "/auth", "/cas/", "/oauth")):
-        return True
-    # 检查页面是否有密码输入框
-    pw_inputs = page.query_selector_all('input[type="password"]')
-    if pw_inputs and len(pw_inputs) > 0:
-        return True
-    return False
-
-
-def read_page(url: str, timeout: float = 60) -> PageResult:
-    """用浏览器打开 URL，返回清洗后的 Markdown。"""
-    ctx = _ensure_browser()
-
-    # 用现有 page 或新建
-    page = ctx.pages[0] if ctx.pages else ctx.new_page()
-
-    page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
-
-    # 如果是登录页，等用户手动登录
-    if _is_login_page(page):
-        log.info("检测到登录页面，请在浏览器窗口中完成登录...")
-        # 等 URL 变化（登录成功后会跳转）或者最多等 120 秒
-        try:
-            page.wait_for_url(
-                lambda u: u != page.url and "login" not in u.lower(),
-                timeout=120_000,
-            )
-            # 等页面加载完
-            page.wait_for_load_state("domcontentloaded", timeout=30_000)
-            log.info(f"登录成功，当前页面: {page.url}")
-        except Exception:
-            log.warning("等待登录超时，尝试继续...")
-
-    # 获取页面内容
-    final_url = page.url
-    html = page.content()
-    title = page.title() or url
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    # 去噪
-    for tag in soup.find_all(["script", "style", "noscript", "iframe", "svg", "nav", "footer", "aside"]):
-        tag.decompose()
-
-    # 找主体
-    main = None
-    for sel in ("article", "main", '[role="main"]', ".content", ".article-content", ".markdown-body", "#content", "#main"):
-        el = soup.select_one(sel)
-        if el and len(el.get_text(strip=True)) > 100:
-            main = el
-            break
-    target = main or soup.body or soup
-
-    links = _extract_links(target, final_url)
-    markdown = _H2T.handle(str(target)).strip()
-
-    if len(markdown) > 200_000:
-        markdown = markdown[:200_000] + "\n\n... [已截断]"
-
-    return PageResult(url=final_url, title=title, content=markdown, links=links)
-
-
-def close_browser():
-    """关闭浏览器（进程退出时调用）。"""
-    global _browser, _context, _pw
-    if _context:
-        try:
-            _context.close()
-        except Exception:
-            pass
-    if _pw:
-        try:
-            _pw.stop()
-        except Exception:
-            pass
-    _context = None
-    _pw = None
-
-
-def _extract_links(soup, base_url: str) -> list[dict]:
-    seen: set[str] = set()
-    links: list[dict] = []
-    for a in soup.find_all("a", href=True):
-        href = a["href"].strip()
-        if not href or href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
-            continue
-        try:
-            absolute = urljoin(base_url, href)
-        except Exception:
-            continue
-        if absolute in seen:
-            continue
-        seen.add(absolute)
-        text = a.get_text(strip=True)[:100] or href
-        links.append({"text": text, "href": absolute})
-    return links