cnks 0.1.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

cnks/extractor.py

@@ -0,0 +1,250 @@
+from typing import Dict, List, Optional, Union
+import logging
+import traceback
+import asyncio
+from contextlib import asynccontextmanager
+from datetime import datetime
+from pydantic import BaseModel
+
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+
+# 配置日志
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    filename="cnki_extractor.log",
+    filemode="a"
+)
+logger = logging.getLogger("cnki_extractor")
+
+# 定义数据模型
+class CNKIContent(BaseModel):
+    """CNKI论文内容模型"""
+    title: str = ""
+    authors: List[str] = []
+    abstract: str = ""
+    keywords: List[str] = []
+    cite_format: str = ""
+    url: str = ""  # 添加URL字段以记录来源
+
+async def get_browser():
+    """获取浏览器实例"""
+    from . import server  # 导入服务器模块以使用查找Chrome的函数
+    
+    playwright = await async_playwright().start()
+    
+    # 查找本地Chrome路径
+    chrome_path = server.find_chrome_executable()
+    
+    if not chrome_path:
+        raise ValueError('未找到Chrome可执行文件，请设置CHROME_PATH环境变量指向Chrome位置')
+    
+    logger.info(f"使用本地Chrome: {chrome_path}")
+    
+    try:
+        # 尝试使用channel='chrome'模式
+        browser = await playwright.chromium.launch(
+            headless=False,
+            channel="chrome"  # 优先使用Chrome通道
+        )
+    except Exception as e:
+        logger.info(f"使用channel='chrome'失败: {str(e)}，尝试使用executable_path")
+        # 如果失败，尝试使用executable_path指定Chrome路径
+        browser = await playwright.chromium.launch(
+            headless=False, 
+            executable_path=chrome_path
+        )
+    
+    context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
+    
+    return playwright, browser, context
+
+async def extract_content_from_url(url: str) -> CNKIContent:
+    """从CNKI页面提取论文内容"""
+    if not url.startswith('https://kns.cnki.net/'):
+        raise ValueError('URL必须是CNKI知网的链接')
+
+    content = CNKIContent(url=url)
+    playwright = None
+    browser = None
+    context = None
+    page = None
+
+    try:
+        # 初始化浏览器
+        playwright, browser, context = await get_browser()
+        
+        # 创建新页面
+        page = await context.new_page()
+        logger.info(f"正在访问页面: {url}")
+
+        try:
+            # 访问页面
+            await page.goto(url, wait_until='networkidle', timeout=60000)
+            logger.info("页面加载完成")
+
+            # 等待并检查登录状态
+            login_text = await page.evaluate('() => document.querySelector(".login-btn")?.textContent || ""')
+            if "登录" in login_text:
+                logger.info("需要登录，请手动登录...")
+                # 等待用户登录完成
+                await asyncio.sleep(10)  # 给用户一些时间登录
+
+            # 提取标题
+            content.title = await page.evaluate('''
+                () => {
+                    const selectors = ['h1.title', '.wx-tit h1', '.title', 'h1'];
+                    for (const selector of selectors) {
+                        const element = document.querySelector(selector);
+                        if (element) {
+                            const text = element.textContent.trim();
+                            if (!text.includes('系统检测')) {
+                                return text.split(/\\s+/)[0];
+                            }
+                        }
+                    }
+                    return "";
+                }
+            ''')
+            logger.info(f"提取到标题: {content.title}")
+
+            # 提取引用格式和作者
+            try:
+                cite_button = await page.wait_for_selector(
+                    'button:has-text("引用"), [class*="cite"], [class*="quote"]',
+                    timeout=15000
+                )
+                if cite_button:
+                    await cite_button.click()
+                    logger.info("获取引用格式")
+                    
+                    cite_result = await page.evaluate('''
+                        () => {
+                            const textarea = document.querySelector('.quote-r textarea.text');
+                            if (textarea) {
+                                const text = textarea.value.trim();
+                                const cite_text = text.replace(/^\\[1\\]/, '').trim();
+                                
+                                const match = cite_text.match(/^([^\\.]+)\\./);
+                                const authors = match ? match[1].split(',').map(a => a.trim()) : [];
+                                
+                                const titleMatch = cite_text.match(/\\.([^\\.]+?)\\[/);
+                                const title = titleMatch ? titleMatch[1].trim() : '';
+                                
+                                return {
+                                    cite_format: cite_text,
+                                    authors: authors,
+                                    title: title
+                                };
+                            }
+                            return null;
+                        }
+                    ''')
+                    
+                    if cite_result:
+                        content.cite_format = cite_result["cite_format"]
+                        content.authors = cite_result["authors"]
+                        if cite_result["title"]:
+                            content.title = cite_result["title"]
+                        logger.info(f"提取到作者: {content.authors}")
+            except Exception as e:
+                logger.error(f"提取引用格式时出错: {str(e)}")
+
+            # 提取摘要
+            content.abstract = await page.evaluate('''
+                () => {
+                    const abstract = document.querySelector('.abstract-text, .abstract, .wx-tit + p');
+                    return abstract ? abstract.textContent.trim() : "";
+                }
+            ''')
+            logger.info(f"提取到摘要长度: {len(content.abstract)} 字符")
+
+            # 提取关键词
+            content.keywords = await page.evaluate('''
+                () => {
+                    const keywordElements = Array.from(document.querySelectorAll('.keywords a, .keywords-text, .keyword'));
+                    if (keywordElements.length > 0) {
+                        return keywordElements.map(k => k.textContent.trim());
+                    }
+                    
+                    const paragraphs = Array.from(document.querySelectorAll('p'));
+                    for (const p of paragraphs) {
+                        if (p.textContent.includes('关键词')) {
+                            const text = p.textContent.trim();
+                            const keywordText = text.split(/关键词[:：]/)[1];
+                            if (keywordText) {
+                                return keywordText.split(/[,，;；]/)
+                                    .map(k => k.trim())
+                                    .filter(k => k);
+                            }
+                        }
+                    }
+                    return [];
+                }
+            ''')
+            logger.info(f"提取到关键词: {content.keywords}")
+            
+            return content
+            
+        except Exception as e:
+            logger.error(f"提取内容时出错: {str(e)}")
+            if page:
+                await page.screenshot(path=f'extraction_error_{datetime.now().strftime("%Y%m%d%H%M%S")}.png')
+            raise Exception(f"提取内容失败: {str(e)}")
+
+    except Exception as e:
+        logger.error(f"处理请求时出错: {str(e)}")
+        logger.error(f"错误堆栈: {traceback.format_exc()}")
+        raise
+    
+    finally:
+        # 关闭资源
+        if page:
+            await page.close()
+        if context:
+            await context.close()
+        if browser:
+            await browser.close()
+        if playwright:
+            await playwright.stop()
+
+async def batch_extract_contents(urls: List[str]) -> List[Dict]:
+    """批量处理多个URL，提取内容并返回JSON格式"""
+    results = []
+    
+    for i, url in enumerate(urls):
+        try:
+            logger.info(f"正在处理第 {i+1}/{len(urls)} 个URL: {url}")
+            content = await extract_content_from_url(url)
+            results.append(content.dict())
+            logger.info(f"成功提取第 {i+1} 个URL的内容")
+            # 添加短暂延迟，避免过快请求导致被封
+            await asyncio.sleep(2)
+        except Exception as e:
+            logger.error(f"处理URL {url} 时出错: {str(e)}")
+            # 添加错误信息而不是跳过，这样可以知道哪些URL处理失败
+            results.append({
+                "url": url,
+                "error": str(e),
+                "title": "",
+                "authors": [],
+                "abstract": "",
+                "keywords": [],
+                "cite_format": ""
+            })
+    
+    return results
+
+# 单元测试
+async def test_extractor():
+    """测试提取器功能"""
+    test_url = "https://kns.cnki.net/kcms2/article/abstract?v=3uoqIhG8C44YLTlOAiTRKibYlV5Vjs7ioT0BO4yQ4m_wBGfVyh51O4GSy-IA63-FppCj3oNSHEUNzY35qnIKlFKtN6Av&uniplatform=NZKPT"
+    try:
+        content = await extract_content_from_url(test_url)
+        print(f"提取成功:\n{content.json(indent=2, ensure_ascii=False)}")
+    except Exception as e:
+        print(f"测试失败: {str(e)}")
+
+if __name__ == "__main__":
+    # 运行测试
+    asyncio.run(test_extractor()) 

cnks/server.py

@@ -8,6 +8,7 @@ import sys
 import time
 import logging
 import webbrowser
+import traceback
 from pathlib import Path
 from urllib.parse import quote
 from typing import Dict, List, Any, Optional, Union
@@ -43,6 +44,16 @@ browser_instance = None
 
 server = Server("cnks")
 
+# 导入我们新创建的extractor模块
+try:
+    from . import chrome_extractor as extractor
+except ImportError:
+    try:
+        import chrome_extractor as extractor
+    except ImportError:
+        extractor = None
+        logger.warning("无法导入chrome_extractor模块，批量提取功能将不可用")
+
 def find_chrome_executable():
     """查找Chrome可执行文件路径"""
     system = platform.system()
@@ -586,31 +597,25 @@ async def handle_get_prompt(
 @server.list_tools()
 async def handle_list_tools() -> list[types.Tool]:
     """列出可用工具"""
-    return [
-        types.Tool(
-            name="search-keywords",
-            description="在知网搜索关键词",
-            inputSchema={
-                "type": "object",
-                "properties": {
-                    "keywords": {"type": "string", "description": "搜索关键词"},
-                },
-                "required": ["keywords"],
-            },
-        ),
-        types.Tool(
-            name="add-note",
-            description="添加笔记",
-            inputSchema={
-                "type": "object",
-                "properties": {
-                    "name": {"type": "string", "description": "笔记名称"},
-                    "content": {"type": "string", "description": "笔记内容"},
+    tools = []
+    
+    # 只添加搜索并提取的组合工具
+    if extractor is not None and PLAYWRIGHT_AVAILABLE:
+        tools.append(
+            types.Tool(
+                name="mcp_cnks_search_and_extract",
+                description="搜索知网关键词并提取所有论文的详细内容",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "keywords": {"type": "string", "description": "搜索关键词"},
+                    },
+                    "required": ["keywords"],
                 },
-                "required": ["name", "content"],
-            },
-        ),
-    ]
+            )
+        )
+    
+    return tools
 
 @server.call_tool()
 async def handle_call_tool(
@@ -619,7 +624,7 @@ async def handle_call_tool(
     """处理工具执行请求"""
     global current_url, page_content
     
-    if name == "search-keywords":
+    if name == "mcp_cnks_search_and_extract" and extractor is not None and PLAYWRIGHT_AVAILABLE:
         if not arguments:
             raise ValueError("缺少参数")
         
@@ -627,76 +632,75 @@ async def handle_call_tool(
         if not keywords:
             raise ValueError("缺少关键词")
         
-        # 优先使用playwright进行搜索
-        if PLAYWRIGHT_AVAILABLE:
+        try:
+            # 第一步：执行搜索
+            logger.info(f"开始执行搜索并提取：关键词 '{keywords}'")
             links_count = await search_with_playwright(keywords)
             current_url = "https://kns.cnki.net/kns8s/search"
             
-            # 返回结构化数据
-            if isinstance(page_content, dict):
+            # 检查搜索结果
+            if not isinstance(page_content, dict) or "links" not in page_content or not page_content["links"]:
                 return [
                     types.TextContent(
                         type="text",
-                        text=json.dumps(page_content, ensure_ascii=False)
+                        text=json.dumps({
+                            "error": "搜索未返回有效链接",
+                            "count": 0,
+                            "results": []
+                        }, ensure_ascii=False)
                     )
                 ]
-            else:
-                # 如果page_content不是字典，返回空结构
-                error_result = {
-                    "count": 0,
-                    "links": [],
-                    "error": "搜索执行失败或结果格式异常"
-                }
+            
+            # 提取链接
+            urls = [link["url"] for link in page_content["links"] if "url" in link]
+            if not urls:
                 return [
                     types.TextContent(
                         type="text",
-                        text=json.dumps(error_result, ensure_ascii=False)
+                        text=json.dumps({
+                            "error": "未找到有效链接",
+                            "count": 0,
+                            "results": []
+                        }, ensure_ascii=False)
                     )
                 ]
-        else:
-            # 如果没有playwright，回退到传统方式
-            result = search_with_direct_chrome(keywords)
-            current_url = "https://kns.cnki.net/kns8s/search"
             
-            # 确保结果是字典格式
-            if not isinstance(result, dict):
-                result = {
-                    "count": 0,
-                    "links": [],
-                    "error": "需要安装playwright以获取链接: uv add playwright"
-                }
-                
+            # 第二步：执行提取
+            logger.info(f"搜索成功，找到 {len(urls)} 个链接，开始提取内容")
+            results = await extractor.batch_extract_contents(urls)
+            
+            # 包装结果
+            result_json = {
+                "keywords": keywords,
+                "count": len(results),
+                "results": results,
+                "success_count": sum(1 for r in results if "error" not in r or not r["error"]),
+                "error_count": sum(1 for r in results if "error" in r and r["error"])
+            }
+            
             return [
                 types.TextContent(
                     type="text",
-                    text=json.dumps(result, ensure_ascii=False)
+                    text=json.dumps(result_json, ensure_ascii=False)
+                )
+            ]
+        except Exception as e:
+            logger.error(f"搜索并提取时出错: {str(e)}")
+            logger.error(traceback.format_exc())
+            return [
+                types.TextContent(
+                    type="text",
+                    text=json.dumps({
+                        "error": f"搜索并提取内容时出错: {str(e)}",
+                        "keywords": keywords,
+                        "count": 0,
+                        "results": []
+                    }, ensure_ascii=False)
                 )
             ]
     
-    elif name == "add-note":
-        if not arguments:
-            raise ValueError("缺少参数")
-        
-        note_name = arguments.get("name")
-        content = arguments.get("content")
-        
-        if not note_name or not content:
-            raise ValueError("缺少名称或内容")
-        
-        # 更新服务器状态
-        notes[note_name] = content
-        
-        # 通知客户端资源已更改
-        await server.request_context.session.send_resource_list_changed()
-        
-        return [
-            types.TextContent(
-                type="text",
-                text=f"已添加笔记 '{note_name}': {content}"
-            )
-        ]
-    
-    raise ValueError(f"未知工具: {name}")
+    else:
+        raise ValueError(f"未知工具: {name}")
 
 async def find_and_count_abstract_links(page):
     """查找并统计包含article/abstract?v=的链接"""
@@ -755,7 +759,7 @@ async def main():
             write_stream,
             InitializationOptions(
                 server_name="cnks",
-                server_version="0.1.1",
+                server_version="0.2.2",
                 capabilities=server.get_capabilities(
                     notification_options=NotificationOptions(),
                     experimental_capabilities={},
@@ -770,34 +774,54 @@ def create_fastmcp_server():
         from mcp.server.fastmcp import FastMCP
         fast_mcp = FastMCP("知网搜索")
         
-        @fast_mcp.tool()
-        async def search_keywords(keywords: str) -> dict:
-            """在知网搜索关键词并返回50个论文链接"""
-            logger.debug("[DEBUG] 正在使用FastMCP的search_keywords函数")
-            if PLAYWRIGHT_AVAILABLE:
-                result_count = await search_with_playwright(keywords)
-                # 返回结构化数据
-                if isinstance(page_content, dict):
-                    return page_content
-                else:
-                    # 如果page_content不是字典，返回空结构
+        # 只添加搜索并提取的工具
+        if extractor is not None and PLAYWRIGHT_AVAILABLE:
+            @fast_mcp.tool()
+            async def mcp_cnks_search_and_extract(keywords: str) -> dict:
+                """搜索关键词并提取所有论文的详细内容"""
+                logger.debug("[DEBUG] 正在使用FastMCP的mcp_cnks_search_and_extract函数")
+                try:
+                    # 第一步：执行搜索
+                    result_count = await search_with_playwright(keywords)
+                    
+                    # 检查搜索结果
+                    if not isinstance(page_content, dict) or "links" not in page_content or not page_content["links"]:
+                        return {
+                            "error": "搜索未返回有效链接",
+                            "keywords": keywords,
+                            "count": 0,
+                            "results": []
+                        }
+                    
+                    # 提取链接
+                    urls = [link["url"] for link in page_content["links"] if "url" in link]
+                    if not urls:
+                        return {
+                            "error": "未找到有效链接",
+                            "keywords": keywords,
+                            "count": 0,
+                            "results": []
+                        }
+                    
+                    # 第二步：执行提取
+                    results = await extractor.batch_extract_contents(urls)
+                    
+                    # 包装结果
+                    return {
+                        "keywords": keywords,
+                        "count": len(results),
+                        "results": results,
+                        "success_count": sum(1 for r in results if "error" not in r or not r["error"]),
+                        "error_count": sum(1 for r in results if "error" in r and r["error"])
+                    }
+                except Exception as e:
+                    logger.error(f"搜索并提取时出错: {str(e)}")
                     return {
+                        "error": f"搜索并提取内容时出错: {str(e)}",
+                        "keywords": keywords,
                         "count": 0,
-                        "links": [],
-                        "error": "搜索执行失败或结果格式异常"
+                        "results": []
                     }
-            else:
-                # 直接Chrome方式不会返回链接
-                return {
-                    "count": 0,
-                    "links": [],
-                    "error": "需要安装playwright以获取链接: uv add playwright"
-                }
-        
-        @fast_mcp.resource("webpage://current")
-        def get_current_webpage() -> str:
-            """获取当前网页内容"""
-            return get_page_content()
         
         return fast_mcp
     except ImportError: