cnks 0.2.5__py3-none-any.whl → 0.3.1__py3-none-any.whl

cnks-0.3.1.dist-info/METADATA

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: cnks
+Version: 0.3.1
+Summary: 中国知网搜索与引文处理系统
+Author-email: bai-z-l <b@iziliang.com>
+Requires-Python: >=3.12
+Requires-Dist: mcp[cli]>=1.6.0
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: python-dotenv>=1.0.0
+Description-Content-Type: text/markdown
+
+# CNKS - 中国知网搜索与引文处理系统
+
+## 简介
+
+CNKS是一个用于搜索中国知网并提取引文数据的工具。该系统能够自动化搜索过程，提取文献信息，并以结构化的方式返回结果。
+
+## 系统架构
+
+CNKS采用服务器-客户端架构，包含以下主要组件：
+
+1. **服务器 (Server)**: 
+   - 处理来自客户端的请求
+   - 按需调用Worker API处理关键词搜索
+   - 管理搜索结果缓存
+
+2. **工作模块 (Worker)**: 
+   - 提供搜索和数据提取API
+   - 使用Playwright自动浏览网页
+   - 解析和提取引文数据
+   - 不再作为独立进程运行，而是由服务器直接调用
+
+3. **客户端 (Client)**:
+   - 命令行界面，用于发送搜索请求
+   - 接收并显示搜索结果
+
+4. **引文处理器 (Citzer)**:
+   - 解析和格式化引文数据
+   - 支持多种引文格式
+
+## 安装
+
+### 要求
+
+- Python 3.12 或更高版本
+- Playwright
+- MCP
+
+### 安装步骤
+
+1. 克隆仓库：
+   ```
+   git clone https://github.com/your-username/cnks.git
+   cd cnks
+   ```
+
+2. 安装依赖：
+   ```
+   pip install -e .
+   playwright install
+   ```
+
+## 使用方法
+
+### 启动服务器
+
+```
+cnks
+```
+或
+```
+cnks-server
+```
+
+### 使用客户端发送请求
+
+```
+cnks-client "搜索关键词"
+```
+
+选项：
+- `--timeout SECONDS`: 设置响应超时时间（默认为60秒）
+
+### 直接测试Worker模块 (仅用于调试)
+
+```
+cnks-worker-test "搜索关键词"
+```
+
+## 配置
+
+系统可通过以下环境变量进行配置：
+
+- `CACHE_FILE`: 缓存文件路径，默认为 "cache.json"
+- `SEARCH_URL`: 搜索URL，默认为中国知网搜索页面
+
+可以创建`.env`文件设置这些环境变量。
+
+## 许可证
+
+[项目许可证信息]

cnks-0.3.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+src/ThisIsAServerSample.py,sha256=p6yhJXMNkFfx1jmeXXXN8UiZ7-h6ZCqx1SqOh1uC5hg,15582
+src/__init__.py,sha256=XSAvnn1ewEm0AssKtCYuKTuT4-LquKU_pBudXzl0v3Q,179
+src/cache.py,sha256=KH9W7vajNsgv_IQn6qOTfDR2w9QrxE3YzR7WO5fVzFU,16270
+src/citzer.py,sha256=9lNmH9pxdxwgG3dZkOVV8jdBzvz2PBS7XCI2Axf8lAg,36548
+src/click50.py,sha256=C3yc4j2a1K5Qf_9-YpJ8qIeyx4sCuSN4rEkgncIOJJ4,22628
+src/client.py,sha256=VlNe-0xv9AqVkqPiFilZxtxC-7zv8UcV4pSGVF6ZFpw,4623
+src/cssci.py,sha256=XDUsbtiDGk2DudbpBl2mP4Z284hKq0YdTBkye1dmdGk,9502
+src/extractlink.py,sha256=qStR6Zo98RpLAmqQxh788rMAW7JN6dKYeSXQLGYC59U,9896
+src/ifverify.py,sha256=SFzcy7G4BRzm8takFBj5LLAl_Py5XauQkIorNey-2cw,4332
+src/main.py,sha256=2zA7LPsC64Ryt2L5q2Gizi7LbzCf3EGQNXU7FobTj5c,1968
+src/searcher.py,sha256=Sa_T4rGgh1chiceSRtY1a7fwi9jGvh2ot2YL1wX8Hes,31566
+src/server.py,sha256=dgULoTjFJLcQc65_H0p55-GiKS6TtTvZUS3zAVKfFt0,17308
+src/worker.py,sha256=O0OwvLKo8QVbvckoiPZXwBdk4dpVel6ePNu02lNbETQ,7549
+cnks-0.3.1.dist-info/METADATA,sha256=SiSrjfscSgQfgHST2qPlkvhyRWh8tBulgGLpBr45pho,2041
+cnks-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cnks-0.3.1.dist-info/entry_points.txt,sha256=ZUeTuZjWMR6j-A-sL2S09SIsc-gFlq6_zl9cvdkypmU,134
+cnks-0.3.1.dist-info/RECORD,,

cnks-0.3.1.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+cnks = src.main:main
+cnks-client = src.client:main
+cnks-server = src.server:main
+cnks-worker-test = src.worker:main

src/ThisIsAServerSample.py

@@ -0,0 +1,377 @@
+import asyncio
+
+from mcp.server.models import InitializationOptions
+import mcp.types as types
+from mcp.server import NotificationOptions, Server
+from pydantic import AnyUrl
+import mcp.server.stdio
+
+server = Server("playwright-server")
+
+@server.list_resources()
+async def handle_list_resources() -> list[types.Resource]:
+    """
+    List available note resources.
+    Each note is exposed as a resource with a custom note:// URI scheme.
+    """
+    return []
+
+@server.read_resource()
+async def handle_read_resource(uri: AnyUrl) -> str:
+    """
+    Read a specific note's content by its URI.
+    The note name is extracted from the URI host component.
+    """
+    raise ValueError(f"Unsupported URI scheme: {uri.scheme}")
+
+
+@server.list_prompts()
+async def handle_list_prompts() -> list[types.Prompt]:
+    """
+    List available prompts.
+    Each prompt can have optional arguments to customize its behavior.
+    """
+    return []
+
+@server.get_prompt()
+async def handle_get_prompt(
+    name: str, arguments: dict[str, str] | None
+) -> types.GetPromptResult:
+    """
+    Generate a prompt by combining arguments with server state.
+    The prompt includes all current notes and can be customized via arguments.
+    """
+    raise ValueError(f"Unknown prompt: {name}")
+
+
+@server.list_tools()
+async def handle_list_tools() -> list[types.Tool]:
+    """
+    List available tools.
+    Each tool specifies its arguments using JSON Schema validation.
+    """
+    return [
+        # types.Tool(
+        #     name="playwright_new_session",
+        #     description="Create a new browser session",
+        #     inputSchema={
+        #         "type": "object",
+        #         "properties": {
+        #             "url": {"type": "string", "description": "Initial URL to navigate to"}
+        #         }
+        #     }
+        # ),
+        types.Tool(
+            name="playwright_navigate",
+            description="Navigate to a URL,thip op will auto create a session",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "url": {"type": "string"}
+                },
+                "required": ["url"]
+            }
+        ),
+        types.Tool(
+            name="playwright_screenshot",
+            description="Take a screenshot of the current page or a specific element",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "selector": {"type": "string", "description": "CSS selector for element to screenshot,null is full page"},
+                },
+                "required": ["name"]
+            }
+        ),
+        types.Tool(
+            name="playwright_click",
+            description="Click an element on the page using CSS selector",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "selector": {"type": "string", "description": "CSS selector for element to click"}
+                },
+                "required": ["selector"]
+            }
+        ),
+        types.Tool(
+            name="playwright_fill",
+            description="Fill out an input field",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "selector": {"type": "string", "description": "CSS selector for input field"},
+                    "value": {"type": "string", "description": "Value to fill"}
+                },
+                "required": ["selector", "value"]
+            }
+        ),
+        types.Tool(
+            name="playwright_evaluate",
+            description="Execute JavaScript in the browser console",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "script": {"type": "string", "description": "JavaScript code to execute"}
+                },
+                "required": ["script"]
+            }
+        ),
+        types.Tool(
+            name="playwright_click_text",
+            description="Click an element on the page by its text content",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "text": {"type": "string", "description": "Text content of the element to click"}
+                },
+                "required": ["text"]
+            }
+        ),
+         types.Tool(
+            name="playwright_get_text_content",
+            description="Get the text content of all elements",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                },
+            }
+        ),
+        types.Tool(
+            name="playwright_get_html_content",
+            description="Get the HTML content of the page",
+             inputSchema={
+                "type": "object",
+                "properties": {
+                    "selector": {"type": "string", "description": "CSS selector for the element"}
+                },
+                "required": ["selector"]
+            }
+        )
+    ]
+
+import uuid
+from playwright.async_api import async_playwright
+import base64
+import os
+
+import asyncio
+
+def update_page_after_click(func):
+    async def wrapper(self, name: str, arguments: dict | None):
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        
+        new_page_future = asyncio.ensure_future(page.context.wait_for_event("page", timeout=3000))
+        
+        result = await func(self, name, arguments)
+        try:
+            new_page = await new_page_future
+            await new_page.wait_for_load_state()
+            self._sessions[session_id]["page"] = new_page
+        except:
+            pass
+            # if page.url != self._sessions[session_id]["page"].url:
+            #     await page.wait_for_load_state()
+            #     self._sessions[session_id]["page"] = page
+        
+        return result
+    return wrapper
+
+class ToolHandler:
+    _sessions: dict[str, any] = {}
+    _playwright: any = None
+
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        raise NotImplementedError
+
+class NewSessionToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        self._playwright = await async_playwright().start()
+        browser = await self._playwright.chromium.launch(headless=False)
+        page = await browser.new_page()
+        session_id = str(uuid.uuid4())
+        self._sessions[session_id] = {"browser": browser, "page": page}
+        url = arguments.get("url")
+        if url:
+            if not url.startswith("http://") and not url.startswith("https://"):
+                url = "https://" + url
+            await page.goto(url)
+        return [types.TextContent(type="text", text="succ")]
+
+class NavigateToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            await NewSessionToolHandler().handle("",{})
+            # return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        url = arguments.get("url")
+        if not url.startswith("http://") and not url.startswith("https://"):
+            url = "https://" + url
+        await page.goto(url)
+        text_content=await GetTextContentToolHandler().handle("",{})
+        return [types.TextContent(type="text", text=f"Navigated to {url}\npage_text_content[:200]:\n\n{text_content[:200]}")]
+
+class ScreenshotToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        name = arguments.get("name")
+        selector = arguments.get("selector")
+        # full_page = arguments.get("fullPage", False)
+        if selector:
+            element = await page.locator(selector)
+            await element.screenshot(path=f"{name}.png")
+        else:
+            await page.screenshot(path=f"{name}.png", full_page=True)
+        with open(f"{name}.png", "rb") as image_file:
+            encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
+        os.remove(f"{name}.png")
+        return [types.ImageContent(type="image", data=encoded_string, mimeType="image/png")]
+
+class ClickToolHandler(ToolHandler):
+    @update_page_after_click
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        selector = arguments.get("selector")
+        await page.locator(selector).click()
+        return [types.TextContent(type="text", text=f"Clicked element with selector {selector}")]
+
+class FillToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        selector = arguments.get("selector")
+        value = arguments.get("value")
+        await page.locator(selector).fill(value)
+        return [types.TextContent(type="text", text=f"Filled element with selector {selector} with value {value}")]
+
+class EvaluateToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        script = arguments.get("script")
+        result = await page.evaluate(script)
+        return [types.TextContent(type="text", text=f"Evaluated script, result: {result}")]
+
+class ClickTextToolHandler(ToolHandler):
+    @update_page_after_click
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        text = arguments.get("text")
+        await page.locator(f"text={text}").nth(0).click()
+        return [types.TextContent(type="text", text=f"Clicked element with text {text}")]
+
+class GetTextContentToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        # text_contents = await page.locator('body').all_inner_texts()
+
+
+        async def get_unique_texts_js(page):
+            unique_texts = await page.evaluate('''() => {
+            var elements = Array.from(document.querySelectorAll('*')); // 先选择所有元素，再进行过滤
+            var uniqueTexts = new Set();
+
+            for (var element of elements) {
+                if (element.offsetWidth > 0 || element.offsetHeight > 0) { // 判断是否可见
+                    var childrenCount = element.querySelectorAll('*').length;
+                    if (childrenCount <= 3) {
+                        var innerText = element.innerText ? element.innerText.trim() : '';
+                        if (innerText && innerText.length <= 1000) {
+                            uniqueTexts.add(innerText);
+                        }
+                        var value = element.getAttribute('value');
+                        if (value) {
+                            uniqueTexts.add(value);
+                        }
+                    }
+                }
+            }
+            //console.log( Array.from(uniqueTexts));
+            return Array.from(uniqueTexts);
+        }
+        ''')
+            return unique_texts
+
+        # 使用示例
+        text_contents = await get_unique_texts_js(page)
+
+
+
+        return [types.TextContent(type="text", text=f"Text content of all elements: {text_contents}")]
+
+class GetHtmlContentToolHandler(ToolHandler):
+    async def handle(self, name: str, arguments: dict | None) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+        if not self._sessions:
+            return [types.TextContent(type="text", text="No active session. Please create a new session first.")]
+        session_id = list(self._sessions.keys())[-1]
+        page = self._sessions[session_id]["page"]
+        selector = arguments.get("selector")
+        html_content = await page.locator(selector).inner_html()
+        return [types.TextContent(type="text", text=f"HTML content of element with selector {selector}: {html_content}")]
+
+
+tool_handlers = {
+    "playwright_navigate": NavigateToolHandler(),
+    "playwright_screenshot": ScreenshotToolHandler(),
+    "playwright_click": ClickToolHandler(),
+    "playwright_fill": FillToolHandler(),
+    "playwright_evaluate": EvaluateToolHandler(),
+    "playwright_click_text": ClickTextToolHandler(),
+    "playwright_get_text_content": GetTextContentToolHandler(),
+    "playwright_get_html_content": GetHtmlContentToolHandler(),
+    "playwright_new_session":NewSessionToolHandler(),
+}
+
+
+@server.call_tool()
+async def handle_call_tool(
+    name: str, arguments: dict | None
+) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+    """
+    Handle tool execution requests.
+    Tools can modify server state and notify clients of changes.
+    """
+    if name in tool_handlers:
+        return await tool_handlers[name].handle(name, arguments)
+    else:
+        raise ValueError(f"Unknown tool: {name}")
+
+async def main():
+    # Run the server using stdin/stdout streams
+    async with mcp.server.stdio.stdio_server() as (read_stream, write_stream):
+        await server.run(
+            read_stream,
+            write_stream,
+            InitializationOptions(
+                server_name="playwright-plus-server",
+                server_version="0.1.0",
+                capabilities=server.get_capabilities(
+                    notification_options=NotificationOptions(),
+                    experimental_capabilities={},
+                ),
+            ),
+        )
+
+if __name__ == "__main__":
+    asyncio.run(main())

src/__init__.py

@@ -0,0 +1,7 @@
+"""
+CNKS - 中国知网搜索与引文处理系统
+
+这是一个基于MCP（模块化通信协议）架构的知网搜索和引文处理系统。
+"""
+
+__version__ = "0.1.0"